Take the parsed stats from the DamageProfiler and add it to the
        basic stats table at the top of the report
498126
def dmgprof_misinc_stats(self, dict_to_plot, readend, substitution):
        

        headers = OrderedDict()
        headers['{}1'.format(readend)] = {
            'id': 'misinc-stats-1st-{}-{}'.format(readend, substitution),
            'title': '{} {} 1st base'.format(readend, substitution),
            'description': '{} 1st base substitution frequency for {}'.format(readend, substitution),
            'max': 100,
            'min': 0,
            'suffix': '%',
            'scale': 'YlGnBu',
            'modify': lambda x: x * 100.0
        }
        headers['{}2'.format(readend)] = {
            'id': 'misinc-stats-2nd-{}-{}'.format(readend, substitution),
            'title': '{} {} 2nd base'.format(readend, substitution),
            'description': '{} 2nd base substitution frequency for {}'.format(readend, substitution),
            'max': 100,
            'min': 0,
            'suffix': '%',
            'scale': 'BuGn',
            'hidden': True,
            'modify': lambda x: x * 100.0
        }


        # Create new small subset dictionary for entries (we need just the first two data (k,v) pairs from each report)
        # Only the first two parts are informative from both 3' and 5' ends of reads (1st, 2nd base damage pattern)
        data = OrderedDict()
        dict_to_add = dict()

        for key in dict_to_plot.keys():
            tmp = dict_to_plot[key]
            pos = [readend + '1', readend + '2']
            strlist = tmp[:2]
            tuples = list(zip(pos,strlist))
            data = dict((x, y) for x, y in tuples)
            #Extract first two elements from list
            dict_to_add[key] = data

        self.general_stats_addcols(dict_to_add,headers)
882695
def parse_samtools_stats(self):
        

        self.samtools_stats = dict()
        for f in self.find_log_files('samtools/stats'):
            parsed_data = dict()
            for line in f['f'].splitlines():
                if not line.startswith("SN"):
                    continue
                sections = line.split("\t")
                field = sections[1].strip()[:-1]
                field = field.replace(' ', '_')
                value = float(sections[2].strip())
                parsed_data[field] = value

            if len(parsed_data) > 0:
                # Work out some percentages
                if 'raw_total_sequences' in parsed_data:
                    for k in list(parsed_data.keys()):
                        if k.startswith('reads_') and k != 'raw_total_sequences' and parsed_data['raw_total_sequences'] > 0:
                            parsed_data['{}_percent'.format(k)] = (parsed_data[k] / parsed_data['raw_total_sequences']) * 100

                if f['s_name'] in self.samtools_stats:
                    log.debug("Duplicate sample name found! Overwriting: {}"
                              .format(f['s_name']))
                self.add_data_source(f, section='stats')
                self.samtools_stats[f['s_name']] = parsed_data

        # Filter to strip out ignored sample names
        self.samtools_stats = self.ignore_samples(self.samtools_stats)

        if len(self.samtools_stats) > 0:

            # Write parsed report data to a file
            self.write_data_file(self.samtools_stats, 'multiqc_samtools_stats')

            # General Stats Table
            stats_headers = OrderedDict()
            stats_headers['error_rate'] = {
                'title': 'Error rate',
                'description': 'Error rate: mismatches (NM) / bases mapped (CIGAR)',
                'min': 0,
                'max': 100,
                'suffix': '%',
                'scale': 'OrRd',
                'format': '{:,.2f}',
                'modify': lambda x: x * 100.0
            }
            stats_headers['non-primary_alignments'] = {
                'title': '{} Non-Primary'.format(config.read_count_prefix),
                'description': 'Non-primary alignments ({})'.format(config.read_count_desc),
                'min': 0,
                'scale': 'PuBu',
                'modify': lambda x: x * config.read_count_multiplier,
                'shared_key': 'read_count'
            }
            stats_headers['reads_mapped'] = {
                'title': '{} Reads Mapped'.format(config.read_count_prefix),
                'description': 'Reads Mapped in the bam file ({})'.format(config.read_count_desc),
                'min': 0,
                'modify': lambda x: x * config.read_count_multiplier,
                'shared_key': 'read_count'
            }
            stats_headers['reads_mapped_percent'] = {
                'title': '% Mapped',
                'description': '% Mapped Reads',
                'max': 100,
                'min': 0,
                'suffix': '%',
                'scale': 'RdYlGn'
            }
            stats_headers['reads_properly_paired_percent'] = {
                'title': '% Proper Pairs',
                'description': '% Properly Paired Reads',
                'max': 100,
                'min': 0,
                'suffix': '%',
                'scale': 'RdYlGn',
                'hidden': True if (max([x['reads_mapped_and_paired'] for x in self.samtools_stats.values()]) == 0) else False
            }
            stats_headers['reads_MQ0_percent'] = {
                'title': '% MapQ 0 Reads',
                'description': '% of Reads that are Ambiguously Placed (MapQ=0)',
                'max': 100,
                'min': 0,
                'suffix': '%',
                'scale': 'OrRd',
                'hidden': True
            }
            stats_headers['raw_total_sequences'] = {
                'title': '{} Total seqs'.format(config.read_count_prefix),
                'description': 'Total sequences in the bam file ({})'.format(config.read_count_desc),
                'min': 0,
                'modify': lambda x: x * config.read_count_multiplier,
                'shared_key': 'read_count'
            }
            self.general_stats_addcols(self.samtools_stats, stats_headers, 'Samtools Stats')

            # Make bargraph plot of mapped/unmapped reads
            self.alignment_section(self.samtools_stats)

            # Make dot plot of counts
            keys = OrderedDict()
            reads = {
                'min': 0,
                'modify': lambda x: float(x) / 1000000.0,
                'suffix': 'M reads',
                'decimalPlaces': 2,
                'shared_key': 'read_count'
            }
            bases = {
                'min': 0,
                'modify': lambda x: float(x) / 1000000.0,
                'suffix': 'M bases',
                'decimalPlaces': 2,
                'shared_key': 'base_count'
            }
            keys['raw_total_sequences'] = dict(reads, **{'title': 'Total sequences'})
            keys['reads_mapped_and_paired'] = dict(reads, **{'title': 'Mapped &amp; paired', 'description': 'Paired-end technology bit set + both mates mapped' })
            keys['reads_properly_paired'] = dict(reads, **{'title': 'Properly paired', 'description': 'Proper-pair bit set'})
            keys['reads_duplicated'] = dict(reads, **{'title': 'Duplicated', 'description': 'PCR or optical duplicate bit set'})
            keys['reads_QC_failed'] = dict(reads, **{'title': 'QC Failed'})
            keys['reads_MQ0'] = dict(reads, **{'title': 'Reads MQ0', 'description': 'Reads mapped and MQ=0'})
            keys['bases_mapped_(cigar)'] = dict(bases, **{'title': 'Mapped bases (CIGAR)', 'description': 'Mapped bases (CIGAR)'})
            keys['bases_trimmed'] = dict(bases, **{'title': 'Bases Trimmed'})
            keys['bases_duplicated'] = dict(bases, **{'title': 'Duplicated bases'})
            keys['pairs_on_different_chromosomes'] = dict(reads, **{'title': 'Diff chromosomes', 'description': 'Pairs on different chromosomes'})
            keys['pairs_with_other_orientation'] = dict(reads, **{'title': 'Other orientation', 'description': 'Pairs with other orientation'})
            keys['inward_oriented_pairs'] = dict(reads, **{'title': 'Inward pairs', 'description': 'Inward oriented pairs'})
            keys['outward_oriented_pairs'] = dict(reads, **{'title': 'Outward pairs', 'description': 'Outward oriented pairs'})

            self.add_section (
                name = 'Alignment metrics',
                anchor = 'samtools-stats',
                description = "This module parses the output from <code>samtools stats</code>. All numbers in millions.",
                plot = beeswarm.plot(self.samtools_stats, keys, {'id': 'samtools-stats-dp'})
            )

        # Return the number of logs that were found
        return len(self.samtools_stats)
1227615
def dedup_general_stats_table(self):
        

        headers = OrderedDict()
        headers['duplication_rate'] = {
            'title': 'Duplication Rate',
            'description': 'Percentage of reads categorised as a technical duplicate',
            'min': 0,
            'max': 100,
            'suffix': '%',
            'scale': 'OrRd',
            'format': '{:,.0f}',
            'modify': lambda x: x * 100.0
        }
        self.general_stats_addcols(self.dedup_data, headers)
68514
def aggregate_stats(stats_files):
  """"""
  all_stats = {}
  for fname in stats_files:
    with tf.gfile.Open(fname) as f:
      stats = json.loads(f.read())
      for k, v in stats.iteritems():
        if k not in all_stats:
          if isinstance(v, list):
            all_stats[k] = []
          else:
            all_stats[k] = 0

        if isinstance(v, list):
          all_stats[k].extend(v)
        else:
          all_stats[k] += v

  stats = all_stats
  ref_coverage = float(stats["total_found_refs"]) / stats["total_original_refs"]
  len_bounds = [0, 2, 10, 100, 1000, 5000, 10000, 20000, 50000, 100000, 1000000]
  len_counts, len_bounds = np.histogram(stats["ref_lengths"], len_bounds)
  len_dist = len_counts.astype(np.float32) / len_counts.sum()
  wiki_coverage = (float(stats["num_wikis_written"]) /
                   stats["total_original_wikis"])
  wikis_skipped_no_ref = (float(stats["wikis_skipped_no_refs"]) /
                          stats["total_original_wikis"])
  wikis_skipped_no_lead = (float(stats["wikis_skipped_short_lead"]) /
                           stats["total_original_wikis"])
  wiki_ref_coverage = [
      float(found) / orig for found, orig
      in zip(stats["wiki_found_refs"], stats["wiki_original_refs"]) if found
  ]
  coverage_bounds = np.arange(21).astype(np.float32) / 20
  coverage_counts, coverage_bounds = np.histogram(wiki_ref_coverage,
                                                  coverage_bounds)
  coverage_dist = coverage_counts.astype(np.float32) / coverage_counts.sum()

  agg_stats = dict(
      total_original_wikis=stats["total_original_wikis"],
      total_original_refs=stats["total_original_refs"],
      wiki_coverage=wiki_coverage,
      wikis_skipped_no_ref=wikis_skipped_no_ref,
      wikis_skipped_no_lead=wikis_skipped_no_lead,
      overall_ref_coverage=ref_coverage,
      per_wiki_ref_coverage_dist=list((coverage_dist * 100).astype(int)),
      per_wiki_ref_coverage_bounds=list((coverage_bounds * 100).astype(int)),
      ref_len_dist=list((len_dist * 100).astype(int)),
      ref_len_bounds=list(len_bounds),
  )
  return agg_stats
51451
def kallisto_general_stats_table(self):
        

        headers = OrderedDict()
        headers['fragment_length'] = {
            'title': 'Frag Length',
            'description': 'Estimated average fragment length',
            'min': 0,
            'suffix': 'bp',
            'scale': 'RdYlGn'
        }
        headers['percent_aligned'] = {
            'title': '% Aligned',
            'description': '% processed reads that were pseudoaligned',
            'max': 100,
            'min': 0,
            'suffix': '%',
            'scale': 'YlGn'
        }
        headers['pseudoaligned_reads'] = {
            'title': '{} Aligned'.format(config.read_count_prefix),
            'description': 'Pseudoaligned reads ({})'.format(config.read_count_desc),
            'min': 0,
            'scale': 'PuRd',
            'modify': lambda x: x * config.read_count_multiplier,
            'shared_key': 'read_count'
        }
        self.general_stats_addcols(self.kallisto_data, headers)
801018
synchronized void printStats() {
        String line1 = String.format("CACHE STATS - Literals: Hits %d/%d (%.1f%%), Inserts %d Evictions %d\n",
                m_literalHits, m_literalQueries, (m_literalHits * 100.0) / m_literalQueries,
                m_literalInsertions, m_literalEvictions);
        String line2 = String.format("CACHE STATS - Plans:    Hits %d/%d (%.1f%%), Inserts %d Evictions %d\n",
                m_planHits, m_planQueries, (m_planHits * 100.0) /m_planQueries,
                m_planInsertions, m_planEvictions);

        System.out.print(line1 + line2);
        System.out.flush();

        // reset these
        m_literalHits = 0;
        m_literalQueries = 0;
        m_literalInsertions = 0;
        m_literalEvictions = 0;
        m_planHits = 0;
        m_planQueries = 0;
        m_planInsertions = 0;
        m_planEvictions = 0;
    }
34991
def update_stats(self, file_result):
        

        for code, count in file_result.violations.items():
            if code not in self.violations:
                self.violations[code] = 0
            self.violations[code] += file_result.violations[code]

            if 'E' in code.upper():
                self.total_errors += file_result.violations[code]
            elif 'W' in code.upper():
                self.total_warnings += file_result.violations[code]
            else:
                self.total_other += file_result.violations[code]
604961
def addSummaryMetrics(self, dict_to_plot):
        

        headers = OrderedDict()

        headers['std'] = {
            'title': 'Read length std. dev.',
            'description': 'Read length std. dev.',
            'suffix': 'bp',
            'scale': 'PuBu',
            'format': '{:,.2f}',
            'shared_key': 'read_length',
            'hidden': True
        }

        headers['median'] = {
            'title': 'Median read length',
            'description': 'Median read length',
            'suffix': 'bp',
            'scale': 'YlGnBu',
            'format': '{:,.2f}',
            'shared_key': 'read_length'
        }
        headers['mean_readlength'] = {
            'title': 'Mean read length',
            'description': 'Mean read length',
            'suffix': 'bp',
            'scale': 'PuBuGn',
            'format': '{:,.2f}',
            'shared_key': 'read_length',
            'hidden': True
        }

        self.general_stats_addcols(dict_to_plot, headers)
1285557
def update(self,
               stats,
               duration=3):
        
        # Build the stats list
        line = ''
        for plugin, attribute in self.plugins_list:
            # Check if the plugin exist and is enable
            if plugin in stats.getPluginsList() and \
               stats.get_plugin(plugin).is_enable():
                stat = stats.get_plugin(plugin).get_export()
            else:
                continue

            # Build the line to display (header or data)
            if self.header:
                line += self.build_header(plugin, attribute, stat)
            else:
                line += self.build_data(plugin, attribute, stat)

        # Display the line (without the last 'separator')
        printandflush(line[:-1])

        # Display header one time
        self.header = False

        # Wait until next refresh
        if duration > 0:
            time.sleep(duration)
1126896
private void addMetaData() {
		document.addTitle(getDocumentTitle());
		document.addAuthor(ReleaseInfoAdapter.getCompany());
		document.addCreator("NextReports " + ReleaseInfoAdapter.getVersionNumber());
		document.addSubject("Created by NextReports Designer" + ReleaseInfoAdapter.getVersionNumber());
		document.addCreationDate();
		document.addKeywords(ReleaseInfoAdapter.getHome());
	}
427261
def append(self, report):
        """"""
        assert report not in self.examples
        self.count += 1
        if len(self.examples) < self.top:
            self.examples.append(report)
852617
def _populate_basic_data(mfg_event, record):
  """"""
  # TODO:
  #   * Missing in proto: set run name from metadata.
  #   * `part_tags` field on proto is unused
  #   * `timings` field on proto is unused.
  #   * Handle arbitrary units as uom_code/uom_suffix.

  # Populate non-repeated fields.
  mfg_event.dut_serial = record.dut_id
  mfg_event.start_time_ms = record.start_time_millis
  mfg_event.end_time_ms = record.end_time_millis
  mfg_event.tester_name = record.station_id
  mfg_event.test_name = record.metadata.get('test_name') or record.station_id
  mfg_event.test_status = test_runs_converter.OUTCOME_MAP[record.outcome]
  mfg_event.operator_name = record.metadata.get('operator_name', '')
  mfg_event.test_version = str(record.metadata.get('test_version', ''))
  mfg_event.test_description = record.metadata.get('test_description', '')

  # Populate part_tags.
  mfg_event.part_tags.extend(record.metadata.get('part_tags', []))

  # Populate phases.
  for phase in record.phases:
    mfg_phase = mfg_event.phases.add()
    mfg_phase.name = phase.name
    mfg_phase.description = phase.codeinfo.sourcecode
    mfg_phase.timing.start_time_millis = phase.start_time_millis
    mfg_phase.timing.end_time_millis = phase.end_time_millis

  # Populate failure codes.
  for details in record.outcome_details:
    failure_code = mfg_event.failure_codes.add()
    failure_code.code = details.code
    failure_code.details = details.description

  # Populate test logs.
  for log_record in record.log_records:
    test_log = mfg_event.test_logs.add()
    test_log.timestamp_millis = log_record.timestamp_millis
    test_log.log_message = log_record.message
    test_log.logger_name = log_record.logger_name
    test_log.levelno = log_record.level
    if log_record.level <= logging.DEBUG:
      test_log.level = test_runs_pb2.TestRunLogMessage.DEBUG
    elif log_record.level <= logging.INFO:
      test_log.level = test_runs_pb2.TestRunLogMessage.INFO
    elif log_record.level <= logging.WARNING:
      test_log.level = test_runs_pb2.TestRunLogMessage.WARNING
    elif log_record.level <= logging.ERROR:
      test_log.level = test_runs_pb2.TestRunLogMessage.ERROR
    elif log_record.level <= logging.CRITICAL:
      test_log.level = test_runs_pb2.TestRunLogMessage.CRITICAL
    test_log.log_source = log_record.source
    test_log.lineno = log_record.lineno
140323
def update_stats(job_stats)
      s = @stats.stats
      current_speed = job_stats.stats[:current_speed] *
                      @stats.penalty(@threads)

      s[:resolution][:completed_records] +=
        job_stats.stats[:resolution][:completed_records]
      @stats.update_eta(current_speed)
      s[:resolution][:time_span] = Time.now - s[:resolution][:start_time]
      s[:matches][7] += job_stats.stats[:matches][7]
    end
955113
def build_stats(counts):
    """"""
    stats = {
        'status': 0,
        'reportnum': counts['reportnum'],
        'title': counts['title'],
        'author': counts['auth_group'],
        'url': counts['url'],
        'doi': counts['doi'],
        'misc': counts['misc'],
    }
    stats_str = "%(status)s-%(reportnum)s-%(title)s-%(author)s-%(url)s-%(doi)s-%(misc)s" % stats
    stats["old_stats_str"] = stats_str
    stats["date"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    stats["version"] = version
    return stats
1050598
function modifyStatsObject ( stats, path ) {
  stats.path = path;
  stats.basename = fs.basename(path);
  stats.dirname = fs.dirname(path);
  stats.extname = fs.extname(path);
  return stats;
}
547969
def statstable(args):
    
    p = OptionParser(statstable.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    gff_files = args
    for metric in metrics:
        logging.debug("Parsing files in `{0}`..".format(metric))

        table = {}
        for x in gff_files:
            pf = op.basename(x).split(".")[0]
            numberfile = op.join(metric, pf + ".txt")
            ar = [int(x.strip()) for x in open(numberfile)]
            sum = SummaryStats(ar).todict().items()
            keys, vals = zip(*sum)
            keys = [(pf, x) for x in keys]
            table.update(dict(zip(keys, vals)))

        print(tabulate(table), file=sys.stderr)
1007112
def update_stats(stats, start_time, data):
    '''
    
    '''
    end_time = time.time()
    cmd = data['cmd']
    # the jid is used as the create time
    try:
        jid = data['jid']
    except KeyError:
        try:
            jid = data['data']['__pub_jid']
        except KeyError:
            log.info('jid not found in data, stats not updated')
            return stats
    create_time = int(time.mktime(time.strptime(jid, '%Y%m%d%H%M%S%f')))
    latency = start_time - create_time
    duration = end_time - start_time

    stats[cmd]['runs'] += 1
    stats[cmd]['latency'] = (stats[cmd]['latency'] * (stats[cmd]['runs'] - 1) + latency) / stats[cmd]['runs']
    stats[cmd]['mean'] = (stats[cmd]['mean'] * (stats[cmd]['runs'] - 1) + duration) / stats[cmd]['runs']

    return stats