Take the parsed stats from the methylQA report and add it to the
        basic stats table at the top of the report
321651
def methylqa_general_stats_table(self):
        

        headers = OrderedDict()
        headers['coverage'] = {
            'title': 'Fold Coverage',
            'min': 0,
            'suffix': 'X',
            'scale': 'YlGn'
        }
        self.general_stats_addcols(self.methylqa_data, headers)
1171474
def report_parse(self):
        
        # Initialise lists
        report_strains = list()
        genus_list = list()
        if self.analysistype == 'mlst':
            for sample in self.runmetadata.samples:
                try:
                    genus_list.append(sample.general.referencegenus)
                except AttributeError:
                    sample.general.referencegenus = 'ND'
                    genus_list.append(sample.general.referencegenus)
        # Read in the report
        if self.analysistype == 'mlst':
            for genus in genus_list:
                try:
                    report_name = os.path.join(self.reportpath, '{at}_{genus}.csv'.format(at=self.analysistype,
                                                                                          genus=genus))
                    report_strains = self.report_read(report_strains=report_strains,
                                                      report_name=report_name)
                except FileNotFoundError:
                    report_name = self.report
                    report_strains = self.report_read(report_strains=report_strains,
                                                      report_name=report_name)
        else:
            report_name = self.report
            report_strains = self.report_read(report_strains=report_strains,
                                              report_name=report_name)
        # Populate strains not in the report with 'empty' GenObject with appropriate attributes
        for sample in self.runmetadata.samples:
            if sample.name not in report_strains:
                setattr(sample, self.analysistype, GenObject())
                sample[self.analysistype].sequencetype = 'ND'
                sample[self.analysistype].matches = 0
                sample[self.analysistype].results = dict()
402391
def add_stats_plot(self):
        """"""

        keys = OrderedDict()
        keys['species_a'] = {'color': '#437bb1', 'name': 'Species a'}
        keys['species_b'] = {'color': '#b1084c', 'name': 'Species b'}
        keys['ambiguous'] = {'color': '#333333', 'name': 'Ambiguous'}

        plot_config = {
            'id': "disambiguated_alignments",
            'title': "Disambiguate: Alignment Counts",
            'cpswitch_counts_label': "# Reads",
            'ylab': "# Reads"
        }

        self.add_section(
            plot=bargraph.plot(self.data, keys, plot_config)
        )
454419
def mlstreporter(self):
        
        logging.info('Writing reports')
        # Initialise variables
        header_row = str()
        combinedrow = str()
        combined_header_row = str()
        reportdirset = set()
        mlst_dict = dict()
        # Populate a set of all the report directories to use. A standard analysis will only have a single report
        # directory, while pipeline analyses will have as many report directories as there are assembled samples
        for sample in self.runmetadata.samples:
            if sample.general.bestassemblyfile != 'NA':
                # Ignore samples that lack a populated reportdir attribute
                if sample[self.analysistype].reportdir != 'NA':
                    make_path(sample[self.analysistype].reportdir)
                    # Add to the set - I probably could have used a counter here, but I decided against it
                    reportdirset.add(sample[self.analysistype].reportdir)
        # Create a report for each sample from :self.resultprofile
        for sample in self.runmetadata.samples:
            if sample.general.bestassemblyfile != 'NA':
                if sample[self.analysistype].reportdir != 'NA':
                    if type(sample[self.analysistype].allelenames) == list:
                        # Initialise the string
                        row = str()
                        if self.analysistype == 'mlst':
                            header_row = str()
                            try:
                                if sample.general.referencegenus not in mlst_dict:
                                    mlst_dict[sample.general.referencegenus] = dict()
                            except AttributeError:
                                sample.general.referencegenus = 'ND'
                                mlst_dict[sample.general.referencegenus] = dict()
                        # Additional fields such as clonal complex and lineage
                        additional_fields = list()
                        #
                        if self.meta_headers:
                            for header in self.meta_headers:
                                try:
                                    _ = sample[self.analysistype].meta_dict[
                                        sample[self.analysistype].sequencetype][header]
                                    additional_fields.append(header.rstrip())
                                except (AttributeError, KeyError):
                                    pass
                        if self.analysistype == 'mlst':
                            additional_fields = sorted(additional_fields)
                            #
                            try:
                                if sample.general.referencegenus == 'Listeria':
                                    additional_fields.append('PredictedSerogroup')
                            except AttributeError:
                                pass
                            header_fields = additional_fields
                        else:
                            additional_fields = [
                                'genus', 'species', 'subspecies', 'lineage', 'sublineage', 'other_designation', 'notes'
                            ]
                            header_fields = [
                                'rMLST_genus', 'species', 'subspecies', 'lineage', 'sublineage', 'other_designation',
                                'notes'
                            ]
                        # Populate the header with the appropriate data, including all the genes in the list of targets
                        if not header_row:
                            if additional_fields:
                                header_row = 'Strain,MASHGenus,{additional},SequenceType,Matches,{matches},\n' \
                                    .format(additional=','.join(header_fields),
                                            matches=','.join(sorted(sample[self.analysistype].allelenames)))
                            else:
                                header_row = 'Strain,MASHGenus,SequenceType,Matches,{matches},\n' \
                                    .format(matches=','.join(sorted(sample[self.analysistype].allelenames)))
                        # Iterate through the best sequence types for the sample
                        for seqtype in self.resultprofile[sample.name]:
                            sample[self.analysistype].sequencetype = seqtype
                            try:
                                if sample.general.referencegenus == 'Listeria':
                                    for serogroup, mlst_list in self.listeria_serogroup_dict.items():
                                        if seqtype in [str(string) for string in mlst_list]:
                                            sample[self.analysistype].meta_dict[seqtype]['PredictedSerogroup'] = \
                                                serogroup
                            except AttributeError:
                                pass
                            # The number of matches to the profile
                            sample[self.analysistype].matches = list(self.resultprofile[sample.name][seqtype].keys())[0]
                            # Extract the closest reference genus
                            try:
                                genus = sample.general.referencegenus
                            except AttributeError:
                                try:
                                    genus = sample.general.closestrefseqgenus
                                except AttributeError:
                                    genus = 'ND'
                            # If this is the first of one or more sequence types, include the sample name
                            if additional_fields:
                                row += '{name},{mashgenus},{additional},{seqtype},{matches},'\
                                    .format(name=sample.name,
                                            mashgenus=genus,
                                            additional=','.join(sample[self.analysistype].
                                                                meta_dict[sample[self.analysistype]
                                                                .sequencetype][header] for header in additional_fields),
                                            seqtype=seqtype,
                                            matches=sample[self.analysistype].matches)
                            else:
                                row += '{name},{mashgenus},{seqtype},{matches},' \
                                    .format(name=sample.name,
                                            mashgenus=genus,
                                            seqtype=seqtype,
                                            matches=sample[self.analysistype].matches)
                            # Iterate through all the genes present in the analyses for the sample
                            for gene in sorted(sample[self.analysistype].allelenames):
                                refallele = sample[self.analysistype].profiledata[seqtype][gene]
                                # Set the allele and percent id from the dictionary's keys and values, respectively
                                allele = \
                                    list(self.resultprofile[sample.name][seqtype][sample[self.analysistype].matches]
                                         [gene].keys())[0]
                                percentid = \
                                    list(self.resultprofile[sample.name][seqtype][sample[self.analysistype].matches]
                                         [gene].values())[0]
                                try:
                                    if refallele and refallele != allele:
                                        if 0 < float(percentid) < 100:
                                            row += '{} ({:.2f}%),'.format(allele, float(percentid))
                                        else:
                                            row += '{} ({}),'.format(allele, refallele)
                                    else:
                                        # Add the allele and % id to the row (only add the % identity if it is not 100%)
                                        if 0 < float(percentid) < 100:
                                            row += '{} ({:.2f}%),'.format(allele, float(percentid))
                                        else:
                                            row += '{},'.format(allele)
                                    self.referenceprofile[sample.name][gene] = allele
                                except ValueError:
                                    pass
                            # Add a newline
                            row += '\n'
                        #
                        combinedrow += row
                        #
                        combined_header_row += header_row
                        combined_header_row += row
                        if self.analysistype == 'mlst':
                            mlst_dict[sample.general.referencegenus]['header'] = header_row
                            try:
                                mlst_dict[sample.general.referencegenus]['combined_row'] += row
                            except KeyError:
                                mlst_dict[sample.general.referencegenus]['combined_row'] = str()
                                mlst_dict[sample.general.referencegenus]['combined_row'] += row
                        # If the length of the # of report directories is greater than 1 (script is being run as part of
                        # the assembly pipeline) make a report for each sample
                        if self.pipeline:
                            # Open the report
                            with open(os.path.join(sample[self.analysistype].reportdir,
                                                   '{}_{}.csv'.format(sample.name, self.analysistype)), 'w') as report:
                                # Write the row to the report
                                report.write(header_row)
                                report.write(row)
        # Create the report folder
        make_path(self.reportpath)
        # Create the report containing all the data from all samples
        if self.analysistype == 'mlst':
            for genus in mlst_dict:
                if mlst_dict[genus]['combined_row']:
                    with open(os.path.join(self.reportpath, '{at}_{genus}.csv'.format(at=self.analysistype,
                                                                                      genus=genus)), 'w') \
                            as mlstreport:
                        # Add the header
                        mlstreport.write(mlst_dict[genus]['header'])
                        # Write the results to this report
                        mlstreport.write(mlst_dict[genus]['combined_row'])
            with open(os.path.join(self.reportpath,  '{at}.csv'.format(at=self.analysistype)), 'w') \
                    as combinedreport:
                # Write the results to this report
                combinedreport.write(combined_header_row)
        else:
            with open(os.path.join(self.reportpath,  '{at}.csv'.format(at=self.analysistype)), 'w') \
                    as combinedreport:
                # Add the header
                combinedreport.write(header_row)
                # Write the results to this report
                combinedreport.write(combinedrow)
1227615
def dedup_general_stats_table(self):
        

        headers = OrderedDict()
        headers['duplication_rate'] = {
            'title': 'Duplication Rate',
            'description': 'Percentage of reads categorised as a technical duplicate',
            'min': 0,
            'max': 100,
            'suffix': '%',
            'scale': 'OrRd',
            'format': '{:,.0f}',
            'modify': lambda x: x * 100.0
        }
        self.general_stats_addcols(self.dedup_data, headers)
219968
def _add_data_to_general_stats(self, data):
    
    headers = _get_general_stats_headers()

    self.general_stats_headers.update(headers)

    header_names = ('ERROR_count', 'WARNING_count', 'file_validation_status')

    general_data = dict()
    for sample in data:
        general_data[sample] = {column: data[sample][column] for column in header_names}
        if sample not in self.general_stats_data:
            self.general_stats_data[sample] = dict()
        if data[sample]['file_validation_status'] != 'pass':
            headers['file_validation_status']['hidden'] = False
        self.general_stats_data[sample].update(general_data[sample])
743509
def process_stats(self, stats, prefix, metric_categories, nested_tags, tags, recursion_level=0):
        
        for child in stats:
            if child.tag in metrics.METRIC_VALUE_FIELDS:
                self.submit_metrics(child, prefix, tags)
            elif child.tag in metrics.CATEGORY_FIELDS:
                recursion_tags = tags + ["{}:{}".format(nested_tags.get(prefix)[recursion_level], child.get('name'))]
                self.process_stats(child, prefix, metric_categories, nested_tags, recursion_tags, recursion_level + 1)
22430
def update(self, stats):
        """"""
        # Get the stats
        all_stats = stats.getAllExportsAsDict(plugin_list=self.plugins_to_export())

        # Init data with timestamp (issue#708)
        if self.first_line:
            csv_header = ['timestamp']
        csv_data = [time.strftime('%Y-%m-%d %H:%M:%S')]

        # Loop over plugins to export
        for plugin in self.plugins_to_export():
            if isinstance(all_stats[plugin], list):
                for stat in all_stats[plugin]:
                    # First line: header
                    if self.first_line:
                        csv_header += ('{}_{}_{}'.format(
                            plugin, self.get_item_key(stat), item) for item in stat)
                    # Others lines: stats
                    csv_data += itervalues(stat)
            elif isinstance(all_stats[plugin], dict):
                # First line: header
                if self.first_line:
                    fieldnames = iterkeys(all_stats[plugin])
                    csv_header += ('{}_{}'.format(plugin, fieldname)
                                   for fieldname in fieldnames)
                # Others lines: stats
                csv_data += itervalues(all_stats[plugin])

        # Export to CSV
        if self.first_line:
            self.writer.writerow(csv_header)
            self.first_line = False
        self.writer.writerow(csv_data)
        self.csv_file.flush()
653942
def parse_gatk_base_recalibrator(self):
        

        report_table_headers = {
            '#:GATKTable:Arguments:Recalibration argument collection values used in this run': 'arguments',
            '#:GATKTable:Quantized:Quality quantization map': 'quality_quantization_map',
            '#:GATKTable:RecalTable0:': 'recal_table_0',
            '#:GATKTable:RecalTable1:': 'recal_table_1',
            '#:GATKTable:RecalTable2:': 'recal_table_2',
        }
        samples_kept = {rt_type: set() for rt_type in recal_table_type}
        self.gatk_base_recalibrator = {recal_type:
                                           {table_name: {}
                                            for table_name
                                            in report_table_headers.values()}
                                       for recal_type in recal_table_type}

        for f in self.find_log_files('gatk/base_recalibrator', filehandles=True):
            parsed_data = self.parse_report(f['f'].readlines(), report_table_headers)
            rt_type = determine_recal_table_type(parsed_data)
            if len(parsed_data) > 0:
                if f['s_name'] in samples_kept[rt_type]:
                    log.debug("Duplicate sample name found! Overwriting: {}".format(f['s_name']))
                else:
                    samples_kept[rt_type].add(f['s_name'])

                self.add_data_source(f, section='base_recalibrator')
                for table_name, sample_tables in parsed_data.items():
                    self.gatk_base_recalibrator[rt_type][table_name][
                        f['s_name']] = sample_tables

        # Filter to strip out ignored sample names
        for rt_type in recal_table_type:
            for table_name, sample_tables in self.gatk_base_recalibrator[rt_type].items():
                self.gatk_base_recalibrator[rt_type][table_name] = self.ignore_samples(
                    sample_tables)

        n_reports_found = sum([len(samples_kept[rt_type]) for rt_type in recal_table_type])

        if n_reports_found > 0:
            log.info("Found {} BaseRecalibrator reports".format(n_reports_found))

            self.add_quality_score_vs_no_of_observations_section()

        return n_reports_found
14199
def report(self):
        
        output = OrderedDict()

        for arg in GLOBAL_ARGUMENTS:
            output[arg] = getattr(self.args, arg) or self.config.get(arg, None)

        output['title'] = getattr(self.args, 'title') or self.config.get('title', 'Untitled Report')
        output['run_date'] = datetime.now().strftime('%Y-%m-%d')
        output['queries'] = []

        for analytic in self.config.get('queries', []):
            print 'Querying "%s"' % analytic['name']

            results = self.query(
                metrics=analytic['metrics'],
                dimensions=analytic.get('dimensions', []),
                filters=analytic.get('filter', None),
                segment=analytic.get('segment', None),
                sort=analytic.get('sort', []),
                start_index=analytic.get('start-index', 1),
                max_results=analytic.get('max-results', 10)
            )
            
            dimensions_len = len(analytic.get('dimensions', []))

            data = OrderedDict([ 
                ('config', analytic),
                ('sampled', results.get('containsSampledData', False)),
                ('sampleSize', int(results.get('sampleSize', 0))),
                ('sampleSpace', int(results.get('sampleSpace', 0))),
                ('data_types', OrderedDict()),
                ('data', OrderedDict())
            ])
                    
            for column in results['columnHeaders'][dimensions_len:]:
                data['data_types'][column['name']] = column['dataType']

            def cast_data_type(d, dt):
                if dt == 'INTEGER':
                    return int(d)
                elif data_type in ['TIME', 'FLOAT', 'CURRENCY', 'PERCENT']:  
                    return float(d)
                else:
                    raise Exception('Unknown metric data type: %s' % data_type)

            for i, metric in enumerate(analytic['metrics']):
                data['data'][metric] = OrderedDict()
                data_type = data['data_types'][metric]

                if dimensions_len:
                    for row in results.get('rows', []):
                        column = i + dimensions_len
                        label = ','.join(row[:dimensions_len]) 
                        value = cast_data_type(row[column], data_type)

                        data['data'][metric][label] = value 

                data['data'][metric]['total'] = cast_data_type(results['totalsForAllResults'][metric], data_type)

                # Prevent rate-limiting
                sleep(1)

            output['queries'].append(data)

        return output
806211
def make_statistics_information(info):
  """"""
  if not info.splits.total_num_examples:
    # That means that we have yet to calculate the statistics for this.
    return "None computed"

  stats = [(info.splits.total_num_examples, "ALL")]
  for split_name, split_info in info.splits.items():
    stats.append((split_info.num_examples, split_name.upper()))
  # Sort reverse on number of examples.
  stats.sort(reverse=True)
  stats = "\n".join([
      "{0:10} | {1:>10,}".format(name, num_exs) for (num_exs, name) in stats
  ])
  return STATISTICS_TABLE.format(split_statistics=stats)
877526
def report(self):
        
        # Add all the genes to the header
        header = 'Sample,'
        data = str()
        with open(os.path.join(self.reportpath, '{at}.csv'.format(at=self.analysistype)), 'w') as report:
            write_header = True
            for sample in self.runmetadata:
                data += sample.name + ','
                # Iterate through all the user-supplied target names
                for target in sorted(self.genes):
                    write_results = False
                    # There was an issue with 'target' not matching 'name' due to a dash being replaced by an underscore
                    # only in 'name'. This will hopefully address this issue
                    target = target.replace('-', '_')
                    if write_header:
                        header += '{target}_match_details,{target},'.format(target=target)
                    for name, identity in sample[self.analysistype].results.items():
                        # Ensure that all dashes are replaced with underscores
                        name = name.replace('-', '_')
                        # If the current target matches the target in the header, add the data to the string
                        if name == target:
                            write_results = True
                            gene_results = '{percent_id}% ({avgdepth} +/- {stddev}),{record},'\
                                .format(percent_id=identity,
                                        avgdepth=sample[self.analysistype].avgdepth[name],
                                        stddev=sample[self.analysistype].standarddev[name],
                                        record=sample[self.analysistype].sequences[target])
                            # Populate the data string appropriately
                            data += gene_results
                    # If the target is not present, write dashes to represent the results and sequence
                    if not write_results:
                        data += '-,-,'
                data += ' \n'
                write_header = False
            header += '\n'
            # Write the strings to the report
            report.write(header)
            report.write(data)
848352
function addDirStats(results) {
        var dirStats = {
            all: 0,
            killed: 0,
            survived: 0,
            ignored: 0,
            untested: 0
        };

        _.forOwn(results, function(result) {
            var stats = result.stats || addDirStats(result).stats;
            dirStats.all += stats.all;
            dirStats.killed += stats.killed;
            dirStats.survived += stats.survived;
            dirStats.ignored += stats.ignored;
            dirStats.untested += stats.untested;
        });

        results.stats = dirStats;
        return results;
    }
60905
def add_query_occurrence(self, report):
        """"""

        initial_millis = int(report['parsed']['stats']['millis'])
        mask = report['queryMask']

        existing_report = self._get_existing_report(mask, report)

        if existing_report is not None:
            self._merge_report(existing_report, report)
        else:
            time = None
            if 'ts' in report['parsed']:
                time = report['parsed']['ts']
            self._reports.append(OrderedDict([
                ('namespace', report['namespace']),
                ('lastSeenDate', time),
                ('queryMask', mask),
                ('supported', report['queryAnalysis']['supported']),
                ('indexStatus', report['indexStatus']),
                ('recommendation', report['recommendation']),
                ('stats', OrderedDict([('count', 1),
                                       ('totalTimeMillis', initial_millis),
                                       ('avgTimeMillis', initial_millis)]))]))
213618
def report_saved(report_stats):
    """"""
    if Settings.verbose:
        report = ''
        truncated_filename = truncate_cwd(report_stats.final_filename)

        report += '{}: '.format(truncated_filename)
        total = new_percent_saved(report_stats)
        if total:
            report += total
        else:
            report += '0%'
        if Settings.test:
            report += ' could be saved.'
        if Settings.verbose > 1:
            tools_report = ', '.join(report_stats.report_list)
            if tools_report:
                report += '\n\t' + tools_report
        print(report)
334509
def report(args):
    
    from jcvi.utils.table import tabulate

    p = OptionParser(report.__doc__)

    types = {"read":      ["padded_start", "padded_end", "orient"],
             "consensus": ["padded_consensus_start", "padded_consensus_end"],
             "quality"  : ["qual_clipping_start", "qual_clipping_end", "align_clipping_start", "align_clipping_end"]
            }
    valid_types = tuple(types.keys())
    p.add_option("--type", default="read", choices=valid_types,
            help="choose report type [default: %default]")

    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    acefile, = args
    ace = Ace.read(must_open(acefile))
    logging.debug('Loaded ace file {0}'.format(acefile))

    for c in ace.contigs:
        print(c.name)
        table = dict()
        if opts.type == "read":
            ps, pe = [], []
            ps = [read.padded_start for read in c.af]
            for i in xrange(1, len(ps)):
                pe.append(ps[i] - ps[i-1])
            pe.append(c.nbases)
            map = dict(zip(ps, pe))
            for i, read in enumerate(c.af):
                values = [str(x) for x in (read.padded_start, map[read.padded_start], read.coru)]
                for i, label in enumerate(types[opts.type]):
                    table[(str(read.name), label)] = values[i]
        elif opts.type == "consensus":
            for read in c.bs:
                values = [str(x) for x in (read.padded_start, read.padded_end)]
                for i, label in enumerate(types[opts.type]):
                    table[(str(read.name), label)] = values[i]
        elif opts.type == "quality":
            for read in c.reads:
                (r1, r2) = (read.rd, read.qa)
                values = [str(x) for x in (r2.qual_clipping_start, r2.qual_clipping_end, r2.align_clipping_start, r2.align_clipping_end)]
                for i, label in enumerate(types[opts.type]):
                    table[(str(r1.name), label)] = values[i]
        print(tabulate(table), "\n")
803519
def translate_summary(data):
    
    headers = sorted(data.get("Headers", []))
    summary = '\\FloatBarrier \n \\section{$NAME} \n'.replace('$NAME', data.get("Title", "table"))
    summary += ' \\begin{table}[!ht] \n \\begin{center}'

    # Set the number of columns
    n_cols = len(headers)
    col_str = "l" + "c" * n_cols
    summary += '\n \\begin{tabular}{$NCOLS} \n'.replace("$NCOLS", col_str)
    spacer = ' &' * n_cols + r'\\[.5em]'

    for header in headers:
        summary += '& $HEADER '.replace('$HEADER', header).replace('%', '\%')
    summary += ' \\\\ \hline \n'

    names = sorted(six.iterkeys(data.get("Data", [])))
    for name in names:
        summary += '\n\n \\textbf{{{}}} {} \n'.format(name, spacer)
        cases = data.get("Data", []).get(name, {})
        for case, c_data in cases.items():
            summary += ' $CASE & '.replace('$CASE', str(case))
            for header in headers:
                h_data = c_data.get(header, "")
                if list is type(h_data) and len(h_data) == 2:
                    summary += (' $H_DATA_0 of $H_DATA_1 &'
                                .replace('$H_DATA_0', str(h_data[0]))
                                .replace('$H_DATA_1', str(h_data[1]))
                                .replace('%', '\%'))
                else:
                    summary += ' $H_DATA &'.replace('$H_DATA', str(h_data)).replace('%', '\%')

            # This takes care of the trailing & that comes from processing the headers.
            summary = summary[:-1] + r' \\'

    summary += '\n \end{tabular} \n \end{center} \n \end{table}\n'
    return summary