Parse the featureCounts log file.
485728
def parse_featurecounts_report (self, f):
        

        file_names = list()
        parsed_data = dict()
        for l in f['f'].splitlines():
            thisrow = list()
            s = l.split("\t")
            if len(s) < 2:
                continue
            if s[0] == 'Status':
                for f_name in s[1:]:
                    file_names.append(f_name)
            else:
                k = s[0]
                if k not in self.featurecounts_keys:
                    self.featurecounts_keys.append(k)
                for val in s[1:]:
                    try:
                        thisrow.append(int(val))
                    except ValueError:
                        pass
            if len(thisrow) > 0:
                parsed_data[k] = thisrow
        # Check that this actually is a featureCounts file, as format and parsing is quite general
        if 'Assigned' not in parsed_data.keys():
            return None
        for idx, f_name in enumerate(file_names):

            # Clean up sample name
            s_name = self.clean_s_name(f_name, f['root'])

            # Reorganised parsed data for this sample
            # Collect total count number
            data = dict()
            data['Total'] = 0
            for k in parsed_data:
                data[k] = parsed_data[k][idx]
                data['Total'] += parsed_data[k][idx]

            # Calculate the percent aligned if we can
            try:
                data['percent_assigned'] = (float(data['Assigned'])/float(data['Total'])) * 100.0
            except (KeyError, ZeroDivisionError):
                pass

            # Add to the main dictionary
            if len(data) > 1:
                if s_name in self.featurecounts_data:
                    log.debug("Duplicate sample name found! Overwriting: {}".format(s_name))
                self.add_data_source(f, s_name)
                self.featurecounts_data[s_name] = data
590573
def parse(logfile, time_format):
  ""

  records = [server.record_from_txt(line, only_elapsed=True,
    time_format=time_format) for line in sys.stdin.readlines()]

  # TODO: make this code better.
  def output(summary):
    width = max([len(p[0]) for p in summary]) + 3
    print '\n'.join([
      "%s%s%s" % (p[0], ' ' * (width - len(p[0])),
        colored(minutes_to_txt(p[1]), 'red')) for p in summary])

  output(server.summarize(records))
1285195
def parse(self, filelike, filename):
        """"""
        self.log = log
        self.source = filelike.readlines()
        src = ''.join(self.source)
        try:
            compile(src, filename, 'exec')
        except SyntaxError as error:
            raise ParseError() from error
        self.stream = TokenStream(StringIO(src))
        self.filename = filename
        self.dunder_all = None
        self.dunder_all_error = None
        self.future_imports = set()
        self._accumulated_decorators = []
        return self.parse_module()
700256
def parse_stats_file(self, file_name):
        
        stats = {}
        try:
            with open(file_name, "r") as fhandle:
                fbuffer = []
                save_buffer = False
                for line in fhandle:
                    line = line.rstrip("\n")
                    line = self._trim(line)
                    if line == "" or line.startswith("#"):
                        continue
                    elif line.endswith("{"):
                        save_buffer = True
                        fbuffer.append(line)
                        continue
                    elif line.endswith("}"):
                        tmp_dict = self._parse_config_buffer(fbuffer)
                        fbuffer = None
                        fbuffer = list()
                        if len(tmp_dict) < 1:
                            continue

                        if tmp_dict["_type"] == "info":
                            stats["info"] = tmp_dict
                        elif tmp_dict["_type"] == "programstatus":
                            stats["programstatus"] = tmp_dict
                        else:
                            entity_type = tmp_dict["_type"]
                            if entity_type not in stats.keys():
                                stats[entity_type] = []

                            stats[entity_type].append(tmp_dict)

                        continue
                    elif save_buffer is True:
                        fbuffer.append(line)

        except Exception as exception:
            self.log.info("Caught exception: %s", exception)

        return stats
1328392
def parse_commits(head, name):
    
    for commit in head.traverse():
        yield {
            '_id': commit.hexsha,
            'repository': name,
            'committed_date': datetime.fromtimestamp(commit.committed_date),
            'committer': {
                'name': commit.committer.name,
                'email': commit.committer.email,
            },
            'authored_date': datetime.fromtimestamp(commit.authored_date),
            'author': {
                'name': commit.author.name,
                'email': commit.author.email,
            },
            'description': commit.message,
            'parent_shas': [p.hexsha for p in commit.parents],
            # we only care about the filenames, not the per-file stats
            'files': list(commit.stats.files),
            'stats': commit.stats.total,
        }
1231508
def _load_type_counts(self):
		""""""

		rel_path = os.path.join(CLTK_DATA_DIR,
                                'old_english',
                                'model',
                                'old_english_models_cltk',
                                'data',
                                'oe.counts')
		path = os.path.expanduser(rel_path)

		self.type_counts = {}

		with open(path, 'r') as infile:
			lines = infile.read().splitlines()
			for line in lines:
				count, word = line.split()
				self.type_counts[word] = int(count)
447337
def _parse_image_multilogs_string(config, ret, repo):
    '''
    
    '''
    image_logs, infos = [], None
    if ret and ret.strip().startswith('{') and ret.strip().endswith('}'):
        pushd = 0
        buf = ''
        for char in ret:
            buf += char
            if char == '{':
                pushd += 1
            if char == '}':
                pushd -= 1
            if pushd == 0:
                try:
                    buf = json.loads(buf)
                except Exception:
                    pass
                image_logs.append(buf)
                buf = ''
        image_logs.reverse()
        # search last layer grabbed
        for l in image_logs:
            if isinstance(l, dict):
                if l.get('status') == 'Download complete' and l.get('id'):
                    infos = _get_image_infos(config, repo)
                    break
    return image_logs, infos
15026
def _parse_names_dict(feature_names):
        """"""
        feature_collection = OrderedDict()
        for feature_name, new_feature_name in feature_names.items():
            if isinstance(feature_name, str) and (isinstance(new_feature_name, str) or
                                                  new_feature_name is ...):
                feature_collection[feature_name] = new_feature_name
            else:
                if not isinstance(feature_name, str):
                    raise ValueError('Failed to parse {}, expected string'.format(feature_name))
                else:
                    raise ValueError('Failed to parse {}, expected string or Ellipsis'.format(new_feature_name))
        return feature_collection
522155
def parse_files(self):
        
        log_re = self.log_format_regex
        log_lines = []
        for log_file in self.matching_files():
            with open(log_file) as f:
                matches = re.finditer(log_re, f.read())
                for match in matches:
                    log_lines.append(match.groupdict())
        return log_lines
1105383
def build_usage_logger(self, log):
        ''''''
        def _log_usage():
            log.debug("{}mb peak memory", peak_memory())
            log.debug("{} pending alignment pairs; {} peak pairs",
                      self.pending_pair_count,
                      self.pending_pair_peak_count)
        return _log_usage
938270
def parse(self):
        """"""
        log.debug('Parsing PocketTopo .TXT file %s ...', self.txtfilename)
        SurveyClass = MergingSurvey if self.merge_duplicate_shots else Survey
        txtobj = None

        with codecs.open(self.txtfilename, 'rb', self.encoding) as txtfile:
            lines = txtfile.read().splitlines()

            # first line is cave name and units
            first_line_re = re.compile(r'^([\w\s]*)\(([\w\s]*),([\w\s]*)')
            first_line = lines.pop(0)
            cave_name, length_units, angle_units = first_line_re.search(first_line).groups()
            cave_name, angle_units = cave_name.strip(), int(angle_units)
            txtobj = TxtFile(cave_name, length_units, angle_units)

            while not lines[0]:
                lines.pop(0)  # skip blanks

            # next block identifies surveys (trip) metadata
            while lines[0].startswith('['):
                toks = lines.pop(0).split(None, 3)
                id, date, declination = toks[:3]
                id = id.strip('[]:')
                date = datetime.strptime(date, '%Y/%m/%d').date()
                declination = float(declination)
                comment = toks[3].strip('"') if len(toks) == 4 else ''
                survey = SurveyClass(id, date, comment, declination, cave_name)
                txtobj.add_survey(survey)

            while not lines[0]:
                lines.pop(0)  # skip blanks

            # finally actual survey data
            while lines:
                line = lines.pop(0).strip()
                if not line:
                    continue

                if '"' in line:
                    line, comment = line.split('"', 1)
                    comment = comment.rstrip('"')
                else:
                    comment = None

                if '[' not in line:
                    # this is either a Reference Point or a zero-length fake shot
                    toks = line.split()
                    if len(toks) != 4:  # ??
                        log.debug('Skipping unrecognized shot:  %s %s', line, '"%s"' % comment if comment else '')
                        continue
                    station, vals = toks[0], list(map(float, toks[1:]))
                    if vals[0] == 0.0:  # fake shot
                        log.debug('Skipping zero-length shot:  %s %s', line, '"%s"' % comment if comment else '')
                    else:  # reference point
                        easting, northing, altitude = vals
                        reference_point = UTMLocation(easting, northing, altitude, comment)
                        log.debug('Reference point:  %s', reference_point)
                        txtobj.add_reference_point(station, reference_point)
                    continue

                line, survey_id = line.split('[')
                survey_id = survey_id.rstrip().rstrip(']')
                toks = line.split()
                from_to, (length, azm, inc) = toks[:-3], (float(tok) for tok in toks[-3:])

                if len(from_to) == 2:
                    from_, to = tuple(from_to)  # shot
                elif len(from_to) == 1:
                    from_, to = from_to[0], None  # splay
                elif not from_to and length == 0.0:
                    continue  # skip junk zero-length placeholder shots
                else:
                    raise Exception()

                shot = Shot([('FROM',from_), ('TO',to), ('LENGTH',length), ('AZM',azm), ('INC',inc), ('COMMENT',comment)])
                txtobj[survey_id].add_shot(shot)

        return txtobj
71000
def process(self):
        
        for filename, warnings in self.warnings.iteritems():

            self.fileCounts[filename] = {}
            fc = self.fileCounts[filename]

            fc["warning_count"] = len(warnings)
            fc["warning_breakdown"] = self._warnCount(warnings)
            self.warningCounts = self._warnCount(warnings,
                warningCount=self.warningCounts)
1026369
private int getLogFileIndex(String fileName) {
		int fileIndex = 0;
		try {
			fileIndex = Integer.parseInt(fileName.substring(fileName
					.lastIndexOf(COUNT_SEPARATOR) + 1));
		} catch (NumberFormatException e) {
			e.printStackTrace();
		}
		return fileIndex;
	}
1268301
private void parseAccessLog(Map<String, Object> config) {
        String filename = (String) config.get("access.filePath");
        if (null == filename || 0 == filename.trim().length()) {
            return;
        }
        try {
            this.ncsaLog = new AccessLogger(filename.trim());
        } catch (Throwable t) {
            FFDCFilter.processException(t, getClass().getName() + ".parseAccessLog", "1", this);
            if (TraceComponent.isAnyTracingEnabled() && tc.isEventEnabled()) {
                Tr.event(tc, "Logging service was unable to open a file: " + filename + "; " + t);
            }
            return;
        }

        // save the NCSA format value
        String format = (String) config.get("access.logFormat");
        this.ncsaLog.setFormat(LogUtils.convertNCSAFormat(format));

        // now check for the maximum file size
        String size = (String) config.get("access.maximumSize");
        if (null != size) {
            // convert from MB to bytes
            if (!this.ncsaLog.setMaximumSize(convertInt(size, 0) * 1048576L)) {
                if (TraceComponent.isAnyTracingEnabled() && tc.isDebugEnabled()) {
                    Tr.debug(tc, "Logging service has invalid access log size: " + size);
                }
            }
        }

        // check for the max backup files
        String backups = (String) config.get("access.maximumBackupFiles");
        if (null != backups) {
            this.ncsaLog.setMaximumBackupFiles(convertInt(backups, 1));
        }

    }
1322502
def parse_skewer_log(self, f):
        
        fh = f['f']
        regexes = {
            'fq1': "Input file:\s+(.+)",
            'fq2': "Paired file:\s+(.+)",
            'r_processed': "(\d+) read|reads pairs? processed",
            'r_short_filtered': "(\d+) \(\s*\d+.\d+%\) short read",
            'r_empty_filtered': "(\d+) \(\s*\d+.\d+%\) empty read",
            'r_avail': "(\d+) \(\s*\d+.\d+%\) read",
            'r_trimmed': "(\d+) \(\s*\d+.\d+%\) trimmed read",
            'r_untrimmed': "(\d+) \(\s*\d+.\d+%\) untrimmed read"
        }
        regex_hist = "\s?(\d+)\s+(\d+)\s+(\d+.\d+)%"

        data = dict()
        for k, v in regexes.items():
            data[k] = 0
        data['fq1'] = None
        data['fq2'] = None
        readlen_dist = OrderedDict()

        for l in fh:
            for k, r in regexes.items():
                match = re.search(r, l)
                if match:
                    data[k] = match.group(1).replace(',', '')

            match = re.search(regex_hist, l)
            if match:
                read_length = int(match.group(1))
                pct_at_rl = float(match.group(3))
                readlen_dist[read_length] = pct_at_rl

        if data['fq1'] is not None:
            s_name = self.clean_s_name(data['fq1'], f['root'])
            if s_name in self.skewer_readlen_dist:
                log.debug("Duplicate sample name found in {}! Overwriting: {}".format(f['fn'], s_name))
            self.add_data_source(f, s_name)
            self.add_skewer_data(s_name, data, f)
            self.skewer_readlen_dist[s_name] = readlen_dist

        if data['fq2'] is not None:
            s_name = self.clean_s_name(data['fq1'], f['root'])
            if s_name in self.skewer_readlen_dist:
                log.debug("Duplicate sample name found in {}! Overwriting: {}".format(f['fn'], s_name))
            self.add_data_source(f, s_name)
            self.add_skewer_data(s_name, data, f)
            self.skewer_readlen_dist[s_name] = readlen_dist
1210278
def _calculate_feature_stats(feature_list, prepared, serialization_file):  # pylint: disable=R0914
    """"""
    # Create feature only list
    feats = [x for x, _ in prepared]  # Label is not necessary

    # Calculate all means / mins / maxs
    means = numpy.mean(feats, 0)
    mins = numpy.min(feats, 0)
    maxs = numpy.max(feats, 0)

    # Calculate, min, max and mean vector for each feature with
    # normalization
    start = 0
    mode = 'w'
    arguments = {'newline': ''}
    if sys.version_info.major < 3:
        mode += 'b'
        arguments = {}
    with open(serialization_file, mode, **arguments) as csvfile:
        spamwriter = csv.writer(csvfile,
                                delimiter=str(';'),
                                quotechar=str('"'),
                                quoting=csv.QUOTE_MINIMAL)
        for feature in feature_list:
            end = start + feature.get_dimension()
            # append the data to the feature class
            feature.mean = numpy.array(means[start:end])
            feature.min = numpy.array(mins[start:end])
            feature.max = numpy.array(maxs[start:end])
            start = end
            for mean, fmax, fmin in zip(feature.mean, feature.max,
                                        feature.min):
                spamwriter.writerow([mean, fmax - fmin])
599482
def parse_logs(log_list, date, machine_name, log_type):
    
    output = []
    count = fail = skip = updated = 0

    # Check things are setup correctly
    try:
        machine = Machine.objects.get(name=machine_name)
    except Machine.DoesNotExist:
        return "ERROR: Couldn't find machine named: %s" % machine_name

    if log_type == "alogger":
        parser = AloggerParser()
    else:
        parser = get_parser(log_type)

    # Process each line
    for line_no, line in enumerate(log_list):
        try:
            data = parser.line_to_dict(line)
        except ValueError:
            output.append("%d: Error reading line" % line_no)
            continue

        # if parser returns None, nothing to do, continue to next line
        if data is None:
            skip += 1
            continue

        # check for required fields
        required_fields = [
            'user', 'project', 'jobid', 'jobname',
            'cpu_usage', 'cores',
            'act_wall_time', 'est_wall_time',
            'mem', 'vmem', 'list_pmem', 'list_mem', 'list_pvmem',
            'ctime', 'qtime', 'etime', 'start',
        ]

        for field in required_fields:
            if field not in data:
                output.append(
                    "line %d: %s field not given." % (line_no, field))
                fail = fail + 1
                continue

        # Process user --> account
        try:
            account = Account.objects.get(
                username=data['user'],
                date_deleted__isnull=True)

        except Account.DoesNotExist:
            # Couldn't find user account - Assign to user None
            output.append(
                "line %d: Couldn't find user account for username=%s."
                % (line_no, data['user']))
            fail += 1
            continue

        except Account.MultipleObjectsReturned:
            output.append(
                "line %d: Username %s has multiple active accounts."
                % (line_no, data['user']))
            fail += 1
            continue

        # Process project
        if data['project'] is None:
            output.append(
                "line %d: Project was not supplied." % (line_no))
            fail += 1
            continue

        try:
            project = Project.objects.get(pid=data['project'])

        except Project.DoesNotExist:
            output.append(
                "line %d: Couldn't find specified project %s"
                % (line_no, data['project']))
            fail += 1
            continue

        # memory calculations
        if machine.mem_per_core:
            avail_mem_per_core = machine.mem_per_core * 1024
            avail_mem_for_job = avail_mem_per_core * data['cores']

            if data['list_pmem'] * data['cores'] > data['list_mem']:
                memory_used_per_core = data['list_pmem']
                memory_used_for_job = data['list_pmem'] * data['cores']
            else:
                memory_used_per_core = data['list_mem'] / data['cores']
                memory_used_for_job = data['list_mem']

            if memory_used_for_job > avail_mem_for_job:
                data['cpu_usage'] = ceil(
                    memory_used_per_core / avail_mem_per_core
                    * data['act_wall_time']
                    * data['cores'])

        # apply scaling factor to cpu_usage
        data['cpu_usage'] = data['cpu_usage'] * machine.scaling_factor

        # Everything is good so add entry
        queue, created = Queue.objects.get_or_create(name=data['queue'])

        try:
            cpujob, created = CPUJob.objects.get_or_create(jobid=data['jobid'])
            cpujob.account = account
            cpujob.username = data['user']
            cpujob.project = project
            cpujob.machine = machine
            cpujob.date = date
            cpujob.queue = queue
            cpujob.cpu_usage = data['cpu_usage']
            cpujob.est_wall_time = data['est_wall_time']
            cpujob.act_wall_time = data['act_wall_time']
            cpujob.mem = data['mem']
            cpujob.vmem = data['vmem']
            cpujob.ctime = data['ctime']
            cpujob.qtime = data['qtime']
            cpujob.etime = data['etime']
            cpujob.start = data['start']
            cpujob.cores = data['cores']
            cpujob.exit_status = data['exit_status']
            cpujob.jobname = data['jobname']
            cpujob.list_mem = data['list_mem']
            cpujob.list_vmem = data['list_vmem']
            cpujob.list_pmem = data['list_pmem']
            cpujob.list_pvmem = data['list_pvmem']
            cpujob.save()

        except Exception as e:
            output.append(
                "line %d: Failed to insert a line  - %s" % (line_no, e))
            fail += 1
            continue

        if created:
            count += 1
        else:
            updated += 1

    summary = (
        'Inserted : %i\nUpdated  : %i\nFailed   : %i\nSkiped   : %i'
        % (count, updated, fail, skip)
    )

    logger.debug('Inserted : %i' % count)
    logger.debug('Updated  : %i' % updated)
    logger.debug('Failed   : %i' % fail)
    logger.debug('Skiped   : %i' % skip)

    return summary, output