Find HOMER findpeaks logs and parse their data
495719
def parse_homer_findpeaks(self):
        

        self.homer_findpeaks = dict()
        for f in self.find_log_files('homer/findpeaks', filehandles=True):
            self.parse_findPeaks(f)

        # Filter to strip out ignored sample names
        self.homer_findpeaks = self.ignore_samples(self.homer_findpeaks)

        if len(self.homer_findpeaks) > 0:

            # Write parsed report data to a file
            self.write_data_file(self.homer_findpeaks, 'multiqc_homer_findpeaks')

            # General Stats Table
            stats_headers = OrderedDict()
            stats_headers['approximate_ip_efficiency'] = {
                'title': '% Efficiency',
                'description': 'Approximate IP efficiency',
                'min': 0,
                'max': 100,
                'suffix': '%',
                'scale': 'RdYlGn'
            }
            stats_headers['total_peaks'] = {
                'title': 'Total Peaks',
                'min': 0,
                'format': '{:,.0f}',
                'scale': 'GnBu'
            }
            stats_headers['expected_tags_per_peak'] = {
                'title': 'Tags/Peak',
                'description': 'Expected tags per peak',
                'min': 0,
                'format': '{:,.0f}',
                'scale': 'PuRd'
            }
            self.general_stats_addcols(self.homer_findpeaks, stats_headers, 'HOMER findpeaks')

        return len(self.homer_findpeaks)
1090915
def get_log_events(awsclient, log_group_name, log_stream_name, start_ts=None):
    
    client_logs = awsclient.get_client('logs')

    request = {
        'logGroupName': log_group_name,
        'logStreamName': log_stream_name
    }
    if start_ts:
        request['startTime'] = start_ts

    # TODO exhaust the events!
    # TODO use all_pages !
    response = client_logs.get_log_events(**request)

    if 'events' in response and response['events']:
        return [{'timestamp': e['timestamp'], 'message': e['message']}
                for e in response['events']]
440153
def parse_hisat2_logs(self, f):
        

        # Regexes
        regexes = {
            'unpaired_total': r"Total(?: unpaired)? reads: (\d+)",
            'unpaired_aligned_none': r"Aligned 0 times?: (\d+) \([\d\.]+%\)",
            'unpaired_aligned_one': r"Aligned 1 time: (\d+) \([\d\.]+%\)",
            'unpaired_aligned_multi': r"Aligned >1 times: (\d+) \([\d\.]+%\)",
    		'paired_total': r"Total pairs: (\d+)",
    		'paired_aligned_none': r"Aligned concordantly or discordantly 0 time: (\d+) \([\d\.]+%\)",
    		'paired_aligned_one': r"Aligned concordantly 1 time: (\d+) \([\d\.]+%\)",
    		'paired_aligned_multi': r"Aligned concordantly >1 times: (\d+) \([\d\.]+%\)",
    		'paired_aligned_discord_one': r"Aligned discordantly 1 time: (\d+) \([\d\.]+%\)",
        }

        # Go through log file line by line
        s_name = f['s_name']
        parsed_data = {}

        for l in f['f']:

            # Attempt in vain to find original hisat2 command, logged by another program
            hscmd = re.search(r"hisat2 .+ -[1U] ([^\s,]+)", l)
            if hscmd:
                s_name = self.clean_s_name(hscmd.group(1), f['root'])
                log.debug("Found a HISAT2 command, updating sample name to '{}'".format(s_name))

            # Run through all regexes
            for k, r in regexes.items():
                match = re.search(r, l)
                if match:
                    parsed_data[k] = int(match.group(1))

            # Overall alignment rate
            overall = re.search(r"Overall alignment rate: ([\d\.]+)%", l)
            if overall:
                parsed_data['overall_alignment_rate'] = float(overall.group(1))

                # Save parsed data
                if s_name in self.hisat2_data:
                    log.debug("Duplicate sample name found! Overwriting: {}".format(s_name))
                self.add_data_source(f, s_name)
                self.hisat2_data[s_name] = parsed_data

                # Reset in case we find more in this log file
                s_name = f['s_name']
                parsed_data = {}
123195
def parse_lines(log_parsers, fileinp):
    """"""
    while 1:
        logentry = fileinp.readline()
        if not logentry:
            break
        elif not logentry.rstrip():
            continue  # skip newlines

        processed = False
        for lp in log_parsers:
            if lp.grok(logentry):
                processed = True
        if not processed:
            # error: none of the logparsers worked on the line
            logger = logging.getLogger('logparser')
            logger.warning(
                #'Could not parse line %s, in file %s >>>%s<<<',
                #fileinp.lineno(), fileinp.filename(), line.rstrip())
                'Could not parse line >>>%s<<<', logentry.rstrip())
            print('Could not parse line >>>%s<<<' % logentry.rstrip())
791048
def parse_reports(self):
    

    # Set up vars
    self.picard_rrbs_metrics = dict()

    # Go through logs and find Metrics
    for f in self.find_log_files('picard/rrbs_metrics', filehandles=True):
        parsed_data = dict()
        s_name = None
        keys = None
        for l in f['f']:
            # New log starting
            if 'CollectRrbsMetrics' in l and 'INPUT' in l:
                s_name = None
                keys = None
                # Pull sample name from input
                fn_search = re.search(r"INPUT(?:=|\s+)(\[?[^\s]+\]?)", l, flags=re.IGNORECASE)
                if fn_search:
                    s_name = os.path.basename(fn_search.group(1).strip('[]'))
                    s_name = self.clean_s_name(s_name, f['root'])
                    parsed_data[s_name] = dict()

            if s_name is not None:
                if 'RrbsSummaryMetrics' in l and '## METRICS CLASS' in l:
                    keys = f['f'].readline().strip("\n").split("\t")
                elif keys:
                    vals = l.strip("\n").split("\t")
                    if len(vals) == len(keys):
                        for i, k in enumerate(keys):
                            try:
                                parsed_data[s_name][k] = float(vals[i])
                            except ValueError:
                                parsed_data[s_name][k] = vals[i]
                    else:
                        s_name = None
                        keys = None

        # Remove empty dictionaries
        for s_name in list(parsed_data.keys()):
            if len(parsed_data[s_name]) == 0:
                parsed_data.pop(s_name, None)

        # Collect parsed data
        for s_name in parsed_data.keys():
            if s_name in self.picard_rrbs_metrics:
                log.debug("Duplicate sample name found in {}! Overwriting: {}".format(f['fn'], s_name))
            self.add_data_source(f, s_name, section='RrbsSummaryMetrics')
            self.picard_rrbs_metrics[s_name] = parsed_data[s_name]

    # Filter to strip out ignored sample names
    self.picard_rrbs_metrics = self.ignore_samples(self.picard_rrbs_metrics)

    if len(self.picard_rrbs_metrics) > 0:

        # Write parsed data to a file
        self.write_data_file(self.picard_rrbs_metrics, 'multiqc_picard_RrbsSummaryMetrics')

        # Add to general stats table
        self.general_stats_headers['PCT_CPG_BASES_CONVERTED'] = {
            'title': '% CpG Methylated',
            'description': 'Percentage of times a CpG cytosine was converted',
            'max': 100,
            'min': 0,
            'suffix': '%',
            'format': '{:,.0f}',
            'scale': 'RdYlGn-rev',
            'modify': lambda x: 100 - self.multiply_hundred(x)
        }
        self.general_stats_headers['PCT_NON_CPG_BASES_CONVERTED'] = {
            'title': '% Non-CpG Methylated',
            'description': 'Percentage of times a non-CpG cytosine was converted',
            'max': 100,
            'min': 0,
            'suffix': '%',
            'format': '{:,.0f}',
            'scale': 'RdYlGn',
            'modify': lambda x: 100 - self.multiply_hundred(x)
        }
        self.general_stats_headers['MEDIAN_CPG_COVERAGE'] = {
            'title': 'Median CpG Cov',
            'description': 'Median coverage of CpG sites',
            'min': 0
        }
        for s_name in self.picard_rrbs_metrics:
            if s_name not in self.general_stats_data:
                self.general_stats_data[s_name] = dict()
            self.general_stats_data[s_name].update( self.picard_rrbs_metrics[s_name] )



        # Make the bar plot of converted bases
        pdata_cpg = dict()
        pdata_noncpg = dict()
        for s_name in self.picard_rrbs_metrics.keys():
            pdata_cpg[s_name] = dict()
            pdata_cpg[s_name]['converted'] = self.picard_rrbs_metrics[s_name]['CPG_BASES_CONVERTED']
            pdata_cpg[s_name]['not_converted'] = self.picard_rrbs_metrics[s_name]['CPG_BASES_SEEN'] - self.picard_rrbs_metrics[s_name]['CPG_BASES_CONVERTED']
            pdata_noncpg[s_name] = dict()
            pdata_noncpg[s_name]['converted'] = self.picard_rrbs_metrics[s_name]['NON_CPG_BASES']
            pdata_noncpg[s_name]['not_converted'] = self.picard_rrbs_metrics[s_name]['NON_CPG_BASES'] - self.picard_rrbs_metrics[s_name]['NON_CPG_CONVERTED_BASES']

        keys = OrderedDict()
        keys['not_converted'] = {'name': 'Unconverted Bases (Methylated)'}
        keys['converted'] = {'name': 'Converted Bases (Unmethylated)'}

        # Config for the plot
        pconfig = {
            'id': 'picard_rrbs_converted_bases_plot',
            'title': 'Picard: Converted Bases',
            'ylab': '# CpG Bases',
            'cpswitch_counts_label': 'Number of Bases',
            'data_labels': [
                {'name': 'CpG', 'ylab': '# CpG Bases'},
                {'name': 'Non-CpG', 'ylab': '# Non-CpG Bases'}
            ]
        }

        self.add_section (
            name = 'RRBS Converted Bases',
            anchor = 'picard-rrbssummary-convertedbases',
            plot = bargraph.plot([pdata_cpg, pdata_noncpg], [keys, keys], pconfig)
        )


        # Make the bar plot of processed reads
        pdata = dict()
        for s_name in self.picard_rrbs_metrics.keys():
            pdata[s_name] = dict()
            pdata[s_name]['with_no_cpg'] = self.picard_rrbs_metrics[s_name]['READS_WITH_NO_CPG']
            pdata[s_name]['ignored_short'] = self.picard_rrbs_metrics[s_name]['READS_IGNORED_SHORT']
            pdata[s_name]['ignored_mismatches'] = self.picard_rrbs_metrics[s_name]['READS_IGNORED_MISMATCHES']
            pdata[s_name]['not_ignored'] = (
                self.picard_rrbs_metrics[s_name]['READS_ALIGNED'] -
                pdata[s_name]['with_no_cpg'] -
                pdata[s_name]['ignored_short'] -
                pdata[s_name]['ignored_mismatches']
            )

        keys = OrderedDict()
        keys['not_ignored'] = {'name': 'Utilised reads'}
        keys['with_no_cpg'] = {'name': 'Ignored (no CpG sites)'}
        keys['ignored_short'] = {'name': 'Ignored (too short)'}
        keys['ignored_mismatches'] = {'name': 'Ignored (exceeded mismatch threshold)'}

        # Config for the plot
        pconfig = {
            'id': 'picard_rrbs_ignored_reads_plot',
            'title': 'Picard: RRBS Read Counts',
            'ylab': '# Reads',
            'cpswitch_counts_label': 'Number of Reads'
        }

        self.add_section (
            name = 'RRBS Read Counts',
            anchor = 'picard-rrbssummary-readcounts',
            plot = bargraph.plot(pdata, keys, pconfig)
        )

    # Return the number of detected samples to the parent module
    return len(self.picard_rrbs_metrics)
797802
def parse_reports(self):
    

    # Set up vars
    self.picard_HsMetrics_data = dict()

    # Go through logs and find Metrics
    for f in self.find_log_files('picard/hsmetrics', filehandles=True):
        parsed_data = dict()
        s_name = None
        keys = None
        commadecimal = None
        for l in f['f']:
            # New log starting
            if 'CalculateHsMetrics' in l or 'CollectHsMetrics' in l and 'INPUT' in l:
                s_name = None
                keys = None

                # Pull sample name from input
                fn_search = re.search(r"INPUT(?:=|\s+)(\[?[^\s]+\]?)", l, flags=re.IGNORECASE)
                if fn_search:
                    s_name = os.path.basename(fn_search.group(1).strip('[]'))
                    s_name = self.clean_s_name(s_name, f['root'])
                    parsed_data[s_name] = dict()

            if s_name is not None:
                if 'HsMetrics' in l and '## METRICS CLASS' in l:
                    keys = f['f'].readline().strip("\n").split("\t")
                elif keys:
                    vals = l.strip("\n").split("\t")
                    if len(vals) == len(keys):
                        j = 'NA'
                        if keys[0] == 'BAIT_SET':
                            j = vals[0]
                        parsed_data[s_name][j] = dict()
                        # Check that we're not using commas for decimal places
                        if commadecimal is None:
                            for i, k in enumerate(keys):
                                if k.startswith('PCT_'):
                                    if ',' in vals[i]:
                                        commadecimal = True
                                    else:
                                        commadecimal = False
                        for i, k in enumerate(keys):
                            try:
                                if commadecimal:
                                    vals[i] = vals[i].replace('.', '')
                                    vals[i] = vals[i].replace(',', '.')
                                parsed_data[s_name][j][k] = float(vals[i])
                            except ValueError:
                                parsed_data[s_name][j][k] = vals[i]
                    else:
                        s_name = None
                        keys = None

        # Remove empty dictionaries
        for s_name in list(parsed_data.keys()):
            for j in parsed_data[s_name].keys():
                if len(parsed_data[s_name][j]) == 0:
                    parsed_data[s_name].pop(j, None)
            if len(parsed_data[s_name]) == 0:
                parsed_data.pop(s_name, None)

        # Manipulate sample names if multiple baits found
        for s_name in parsed_data.keys():
            for j in parsed_data[s_name].keys():
                this_s_name = s_name
                if(len(parsed_data[s_name]) > 1):
                    this_s_name = "{}: {}".format(s_name, j)
                if this_s_name in self.picard_HsMetrics_data:
                    log.debug("Duplicate sample name found in {}! Overwriting: {}".format(f['fn'], this_s_name))
                self.add_data_source(f, this_s_name, section='HsMetrics')
                self.picard_HsMetrics_data[this_s_name] = parsed_data[s_name][j]


    # Filter to strip out ignored sample names
    self.picard_HsMetrics_data = self.ignore_samples(self.picard_HsMetrics_data)

    if len(self.picard_HsMetrics_data) > 0:

        # Write parsed data to a file
        self.write_data_file(self.picard_HsMetrics_data, 'multiqc_picard_HsMetrics')

        # Add to general stats table
        # Swap question marks with -1
        data = self.picard_HsMetrics_data
        for s_name in data:
            if data[s_name]['FOLD_ENRICHMENT'] == '?':
                data[s_name]['FOLD_ENRICHMENT'] = -1

        self.general_stats_headers['FOLD_ENRICHMENT'] = {
            'title': 'Fold Enrichment',
            'min': 0,
            'format': '{:,.0f}',
            'scale': 'Blues',
        }
        try:
            covs = config.picard_config['general_stats_target_coverage']
            assert type(covs) == list
            assert len(covs) > 0
            covs = [str(i) for i in covs]
            log.debug("Custom Picard coverage thresholds: {}".format(", ".join([i for i in covs])))
        except (AttributeError, TypeError, AssertionError):
            covs = ['30']
        for c in covs:
            self.general_stats_headers['PCT_TARGET_BASES_{}X'.format(c)] = {
                'id': 'picard_target_bases_{}X'.format(c),
                'title': 'Target Bases {}X'.format(c),
                'description': 'Percent of target bases with coverage &ge; {}X'.format(c),
                'max': 100,
                'min': 0,
                'suffix': '%',
                'format': '{:,.0f}',
                'scale': 'RdYlGn',
                'modify': lambda x: self.multiply_hundred(x)
            }
        for s_name in data:
            if s_name not in self.general_stats_data:
                self.general_stats_data[s_name] = dict()
            self.general_stats_data[s_name].update( data[s_name] )
        data_table = _clean_table(data)
        self.add_section (
                name = 'HSMetrics',
                anchor = 'picard_hsmetrics',
                plot = table.plot(data_table, _get_headers(data_table))
        )
        tbases = _add_target_bases(data)
        self.add_section (
            name = tbases['name'],
            anchor = tbases['anchor'],
            description = tbases['description'],
            plot = tbases['plot']
        )
        hs_pen = _add_hs_penalty(data)
        if hs_pen is not None:
            self.add_section (
                name = hs_pen['name'],
                anchor = hs_pen['anchor'],
                description = hs_pen['description'],
                plot = hs_pen['plot']
            )

    # Return the number of detected samples to the parent module
    return len(self.picard_HsMetrics_data)
1145120
def parse_flash_log(self, logf):
        """"""
        data = OrderedDict()
        samplelogs = self.split_log(logf['f'])
        for slog in samplelogs:
            try:
                sample = dict()
                ## Sample name ##
                s_name = self.clean_pe_name(slog, logf['root'])
                if s_name is None:
                    continue
                sample['s_name'] = s_name

                ## Log attributes ##
                sample['totalpairs'] = self.get_field('Total pairs', slog)
                sample['discardpairs'] = self.get_field('Discarded pairs', slog)
                sample['percdiscard'] = self.get_field('Percent Discarded', slog, fl=True)
                sample['combopairs'] = self.get_field('Combined pairs', slog)
                sample['inniepairs'] = self.get_field('Innie pairs', slog)
                sample['outiepairs'] = self.get_field('Outie pairs', slog)
                sample['uncombopairs'] = self.get_field('Uncombined pairs', slog)
                sample['perccombo'] = self.get_field('Percent combined', slog, fl=True)

                data[s_name] = sample
            except Exception as err:
                log.warning("Error parsing record in {}. {}".format(logf['fn'], err))
                log.debug(traceback.format_exc())
                continue
        return data
440500
def _logstash(url, data):
    '''
    
    '''
    result = salt.utils.http.query(
        url,
        'POST',
        header_dict=_HEADERS,
        data=salt.utils.json.dumps(data),
        decode=True,
        status=True,
        opts=__opts__
    )
    return result
815213
def parse_reports(self):
    

    # Set up vars
    self.picard_pcrmetrics_data = dict()
    self.picard_pcrmetrics_samplestats = dict()

    # Go through logs and find Metrics
    for f in self.find_log_files('picard/pcr_metrics', filehandles=True):
        s_name = None
        for l in f['f']:
            # New log starting
            if 'TargetedPcrMetrics' in l and 'INPUT' in l:
                s_name = None
                # Pull sample name from input
                fn_search = re.search(r"INPUT(?:=|\s+)(\[?[^\s]+\]?)", l, flags=re.IGNORECASE)
                if fn_search:
                    s_name = os.path.basename(fn_search.group(1).strip('[]'))
                    s_name = self.clean_s_name(s_name, f['root'])

            if s_name is not None:
                if 'TargetedPcrMetrics' in l and '## METRICS CLASS' in l:
                    keys = f['f'].readline().strip("\n").split("\t")
                    vals = f['f'].readline().strip("\n").split("\t")
                    if len(vals) == len(keys):
                        if s_name in self.picard_pcrmetrics_data:
                            log.debug("Duplicate sample name found in {}! Overwriting: {}".format(f['fn'], s_name))
                        self.add_data_source(f, s_name, section='TargetedPcrMetrics')
                        self.picard_pcrmetrics_data[s_name] = dict()
                        for i, k in enumerate(keys):
                            try:
                                # Multiply percentages by 100
                                if k.startswith('PCT_'):
                                    vals[i] = float(vals[i]) * 100.0
                                self.picard_pcrmetrics_data[s_name][k] = float(vals[i])
                            except ValueError:
                                self.picard_pcrmetrics_data[s_name][k] = vals[i]

    # Filter to strip out ignored sample names
    self.picard_pcrmetrics_data = self.ignore_samples(self.picard_pcrmetrics_data)

    if len(self.picard_pcrmetrics_data) > 0:

        # Write parsed data to a file
        self.write_data_file(self.picard_pcrmetrics_data, 'multiqc_picard_pcrmetrics')

        # Add to general stats table
        self.general_stats_headers['PCT_AMPLIFIED_BASES'] = {
            'title': '% Amplified Bases',
            'description': 'The fraction of aligned bases that mapped to or near an amplicon.',
            'min': 0,
            'max': 100,
            'suffix': '%',
            'scale': 'BrBG'
        }
        self.general_stats_headers['MEDIAN_TARGET_COVERAGE'] = {
            'title': 'Median Target Coverage',
            'description': 'The median coverage of reads that mapped to target regions of an experiment.',
            'min': 0,
            'suffix': 'X',
            'scale': 'GnBu',
        }

        for s_name in self.picard_pcrmetrics_data:
            if s_name not in self.general_stats_data:
                self.general_stats_data[s_name] = dict()
            self.general_stats_data[s_name].update( self.picard_pcrmetrics_data[s_name] )

        # Bar plot of ignored bases
        keys = OrderedDict()
        keys['ON_AMPLICON_BASES'] = {'name': 'On-amplicon bases'}
        keys['NEAR_AMPLICON_BASES'] = {'name': 'Near-amplicon bases'}
        keys['OFF_AMPLICON_BASES'] = {'name': 'Off-amplicon bases', 'color': '#f28f43'}

        # Config for the plot
        pconfig = {
            'id': 'picard_pcr_metrics_bases',
            'title': 'Picard: PCR Amplicon Bases',
            'ylab': '# Bases',
            'cpswitch_counts_label': '# Bases',
            'hide_zero_cats': False
        }

        self.add_section (
            name = 'PCR Amplicon Bases',
            anchor = 'picard-pcrmetrics-bases',
            description = 'Metrics about reads obtained from targeted PCR experiments.',
            helptext = '''
            This plot shows the number of bases aligned on or near to amplified regions of the genome.

            * `ON_AMPLICON_BASES`: The number of `PF_BASES_ALIGNED` that mapped to an amplified region of the genome.
            * `NEAR_AMPLICON_BASES`: The number of `PF_BASES_ALIGNED` that mapped to within a fixed interval of an amplified region, but not on a baited region.
            * `OFF_AMPLICON_BASES`: The number of `PF_BASES_ALIGNED` that mapped neither on or near an amplicon.

            For more information see the [Picard documentation](https://broadinstitute.github.io/picard/picard-metric-definitions.html#TargetedPcrMetrics).''',
            plot = bargraph.plot(self.picard_pcrmetrics_data, keys, pconfig)
        )


    # Return the number of detected samples to the parent module
    return len(self.picard_pcrmetrics_data)
1277113
def extract_deps(bundles, log=None):
    """"""
    def _flatten(bundle):
        deps = []
        if hasattr(bundle, 'npm'):
            deps.append(bundle.npm)
        for content in bundle.contents:
            if isinstance(content, BundleBase):
                deps.extend(_flatten(content))
        return deps

    flatten_deps = []
    for bundle in bundles:
        flatten_deps.extend(_flatten(bundle))

    packages = defaultdict(list)
    for dep in flatten_deps:
        for pkg, version in dep.items():
            packages[pkg].append(version)

    deps = {}
    for package, versions in packages.items():
        deps[package] = semver.max_satisfying(versions, '*', True)

        if log and len(versions) > 1:
            log('Warn: {0} version {1} resolved to: {2}'.format(
                repr(package), versions, repr(deps[package])
            ))

    return deps
573034
function logHeaders(logger, headers) {
	return _.each(_.keys(headers), function (key) {
		logger.info(key + ': ' + headers[key]);
	});
}
409545
def extract_logs(self, fname, prg):
        
        op = []
        with open(fname, 'r') as f:
            for line in f:
                if prg in line:
                    op.append(line)
        return op
1052641
def searchlog(self, argv):
        """"""
        opts = cmdline(argv, FLAGS_SEARCHLOG)
        self.foreach(opts.args, lambda job: 
            output(job.searchlog(**opts.kwargs)))
518138
def find_message(self):
        ''''''
        while True:
            self.msg = self.mlog.recv_match(condition=args.condition)
            if self.msg is not None and self.msg.get_type() != 'BAD_DATA':
                break
            if self.mlog.f.tell() > self.filesize - 10:
                self.paused = True
                break
        self.last_timestamp = getattr(self.msg, '_timestamp')
1223358
def parse_logs(self, f):
        """"""
        data = {}
        for l in f.splitlines():
            # catch empty lines
            if len(l) == 0:
                continue
            s = l.split("\t")
            data_ = []
            # catch lines with descriptive content: "Of pairs used:"
            for i in s[1:]:
                if len(i) == 0:
                    continue
                try:
                    i.replace('(', '')
                    i.replace(')', '')
                    i.replace(',', '')

                    data_.append(float(i))
                except ValueError:
                    data_.append(i)
            if len(data_) == 0:
                continue
            if s[0].startswith('short range'):
                s[0] = 'short range'
            elif s[0].startswith('same fragment'):
                s[0] = 'same fragment'
            s[0] = s[0].capitalize()
            data[s[0]] = data_
        return data
1212392
def logs(self):
        """"""
        if not self.parent.loaded: self.parent.load()
        logs = self.parent.p.logs_dir.flat_directories
        logs.sort(key=lambda x: x.mod_time)
        return logs
375341
def parse_cutadapt_logs(self, f):
        
        fh = f['f']
        regexes = {
            '1.7': {
                'bp_processed': "Total basepairs processed:\s*([\d,]+) bp",
                'bp_written': "Total written \(filtered\):\s*([\d,]+) bp",
                'quality_trimmed': "Quality-trimmed:\s*([\d,]+) bp",
                'r_processed': "Total reads processed:\s*([\d,]+)",
                'r_with_adapters': "Reads with adapters:\s*([\d,]+)"
            },
            '1.6': {
                'r_processed': "Processed reads:\s*([\d,]+)",
                'bp_processed': "Processed bases:\s*([\d,]+) bp",
                'r_trimmed': "Trimmed reads:\s*([\d,]+)",
                'quality_trimmed': "Quality-trimmed:\s*([\d,]+) bp",
                'bp_trimmed': "Trimmed bases:\s*([\d,]+) bp",
                'too_short': "Too short reads:\s*([\d,]+)",
                'too_long': "Too long reads:\s*([\d,]+)",
            }
        }
        s_name = None
        cutadapt_version = '1.7'
        log_section = None
        for l in fh:
            # New log starting
            if 'cutadapt' in l:
                s_name = None
                c_version = re.match(r'This is cutadapt ([\d\.]+)', l)
                if c_version:
                    try:
                        assert(StrictVersion(c_version.group(1)) <= StrictVersion('1.6'))
                        cutadapt_version = '1.6'
                    except:
                        cutadapt_version = '1.7'
                c_version_old = re.match(r'cutadapt version ([\d\.]+)', l)
                if c_version_old:
                    try:
                        assert(StrictVersion(c_version.group(1)) <= StrictVersion('1.6'))
                        cutadapt_version = '1.6'
                    except:
                        # I think the pattern "cutadapt version XX" is only pre-1.6?
                        cutadapt_version = '1.6'
            # Get sample name from end of command line params
            if l.startswith('Command line parameters'):
                s_name = l.split()[-1]
                # Manage case where sample name is '-' (reading from stdin)
                if s_name == '-':
                    s_name = f['s_name']
                else:
                    s_name = self.clean_s_name(s_name, f['root'])
                if s_name in self.cutadapt_data:
                    log.debug("Duplicate sample name found! Overwriting: {}".format(s_name))
                self.cutadapt_data[s_name] = dict()

            if s_name is not None:
                self.add_data_source(f, s_name)

                # Search regexes for overview stats
                for k, r in regexes[cutadapt_version].items():
                    match = re.search(r, l)
                    if match:
                        self.cutadapt_data[s_name][k] = int(match.group(1).replace(',', ''))

                # Starting a new section
                if '===' in l:
                    log_section = l.strip().strip('=').strip()

                # Histogram showing lengths trimmed
                if 'length' in l and 'count' in l and 'expect' in l:
                    plot_sname = s_name
                    if log_section is not None:
                        plot_sname = '{} - {}'.format(s_name, log_section)
                    self.cutadapt_length_counts[plot_sname] = dict()
                    self.cutadapt_length_exp[plot_sname] = dict()
                    self.cutadapt_length_obsexp[plot_sname] = dict()
                    # Nested loop to read this section while the regex matches
                    for l in fh:
                        r_seqs = re.search("^(\d+)\s+(\d+)\s+([\d\.]+)", l)
                        if r_seqs:
                            a_len = int(r_seqs.group(1))
                            self.cutadapt_length_counts[plot_sname][a_len] = int(r_seqs.group(2))
                            self.cutadapt_length_exp[plot_sname][a_len] = float(r_seqs.group(3))
                            if float(r_seqs.group(3)) > 0:
                                self.cutadapt_length_obsexp[plot_sname][a_len] = float(r_seqs.group(2)) / float(r_seqs.group(3))
                            else:
                                # Cheating, I know. Infinity is difficult to plot.
                                self.cutadapt_length_obsexp[plot_sname][a_len] = float(r_seqs.group(2))
                        else:
                            break

        # Calculate a few extra numbers of our own
        for s_name, d in self.cutadapt_data.items():
            if 'bp_processed' in d and 'bp_written' in d:
                self.cutadapt_data[s_name]['percent_trimmed'] = (float(d['bp_processed'] - d['bp_written']) / d['bp_processed']) * 100
            elif 'bp_processed' in d and 'bp_trimmed' in d:
                self.cutadapt_data[s_name]['percent_trimmed'] = ((float(d.get('bp_trimmed', 0)) + float(d.get('quality_trimmed', 0))) / d['bp_processed']) * 100