Parses sambamba markdup output, returns series with values.

        :param str stats_file: sambamba output file with duplicate statistics.
541339
def parse_duplicate_stats(self, stats_file):
        
        import pandas as pd
        series = pd.Series()
        try:
            with open(stats_file) as handle:
                content = handle.readlines()  # list of strings per line
        except:
            return series
        try:
            line = [i for i in range(len(content)) if "single ends (among them " in content[i]][0]
            series["single-ends"] = re.sub("\D", "", re.sub("\(.*", "", content[line]))
            line = [i for i in range(len(content)) if " end pairs...   done in " in content[i]][0]
            series["paired-ends"] = re.sub("\D", "", re.sub("\.\.\..*", "", content[line]))
            line = [i for i in range(len(content)) if " duplicates, sorting the list...   done in " in content[i]][0]
            series["duplicates"] = re.sub("\D", "", re.sub("\.\.\..*", "", content[line]))
        except IndexError:
            pass
        return series
348424
def ParseMultiple(self, stats, file_objs, kb):
    """"""
    fileset = {stat.pathspec.path: obj for stat, obj in zip(stats, file_objs)}
    return self.ParseFileset(fileset)
361990
def make_frozen_stats_tree(stats):
    
    tree, stats_tree = [], [(None, stats)]
    for x in itertools.count():
        try:
            parent_offset, _stats = stats_tree[x]
        except IndexError:
            break
        stats_tree.extend((x, s) for s in _stats)
        members = (_stats.name, _stats.filename, _stats.lineno,
                   _stats.module, _stats.own_hits, _stats.deep_time)
        tree.append((parent_offset, members))
    return tree
292198
def stats(cmd_options={})
      raw = hg(["log -r ? --stat --template '{node}\n'", hash_id], cmd_options)
      result = raw.scan(/(\d+) files changed, (\d+) insertions\(\+\), (\d+) deletions\(\-\)$/).flatten.map{|r| r.to_i}
      return {} if result.empty? # that commit has no stats
      {
        'files'     => result[0],
        'additions' => result[1],
        'deletions' => result[2],
        'total'     => result[1] + result[2]
      }
    end
296824
def summary(args):
    
    from jcvi.utils.cbook import SummaryStats

    p = OptionParser(summary.__doc__)
    p.add_option("--prefix", help="Generate per block stats [default: %default]")
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    anchorfile, = args
    ac = AnchorFile(anchorfile)
    clusters = ac.blocks
    if clusters == [[]]:
        logging.debug("A total of 0 anchor was found. Aborted.")
        raise ValueError("A total of 0 anchor was found. Aborted.")

    nclusters = len(clusters)
    nanchors = [len(c) for c in clusters]
    nranchors = [_score(c) for c in clusters]  # non-redundant anchors
    print("A total of {0} (NR:{1}) anchors found in {2} clusters.".\
                  format(sum(nanchors), sum(nranchors), nclusters), file=sys.stderr)
    print("Stats:", SummaryStats(nanchors), file=sys.stderr)
    print("NR stats:", SummaryStats(nranchors), file=sys.stderr)

    prefix = opts.prefix
    if prefix:
        pad = len(str(nclusters))
        for i, c in enumerate(clusters):
            block_id = "{0}{1:0{2}d}".format(prefix, i + 1, pad)
            print("\t".join((block_id, str(len(c)))))
326702
def _get_stats_files(data, out_dir=None):
    
    if not out_dir:
        out_dir = utils.safe_makedir(os.path.join(dd.get_work_dir(data),
                                                  "qc", dd.get_sample_name(data), "samtools"))
    stats_file = tz.get_in(["depth", "samtools", "stats"], data)
    idxstats_file = tz.get_in(["depth", "samtools", "idxstats"], data)
    if not stats_file:
        stats_file = os.path.join(out_dir, "%s.txt" % dd.get_sample_name(data))
    if not idxstats_file:
        idxstats_file = os.path.join(out_dir, "%s-idxstats.txt" % dd.get_sample_name(data))
    return stats_file, idxstats_file
1005424
def read_mash_output(result_file):
    
    with open(result_file) as handle:
        lines = handle.readlines()
    mash_results = list()
    for line in lines:
        result = MashResult(line)
        mash_results.append(result)
    return mash_results
792786
def parse_samtools_rmdup(self):
        

        self.samtools_rmdup = dict()
        for f in self.find_log_files('samtools/rmdup', filehandles=True):
            # Example below:
            # [bam_rmdupse_core] 26602816 / 103563641 = 0.2569 in library '   '
            dups_regex = "\[bam_rmdups?e?_core\] (\d+) / (\d+) = (\d+\.\d+) in library '(.*)'"
            s_name = f['s_name']
            for l in f['f']:
                match = re.search(dups_regex, l)
                if match:
                    library_name = match.group(4).strip()
                    if library_name != '':
                        s_name = library_name
                    if s_name in self.samtools_rmdup:
                        log.debug("Duplicate sample name found in {}! Overwriting: {}".format(f['fn'], s_name))
                    self.add_data_source(f, s_name)
                    self.samtools_rmdup[s_name] = dict()
                    self.samtools_rmdup[s_name]['n_dups'] = int(match.group(1))
                    self.samtools_rmdup[s_name]['n_tot'] = int(match.group(2))
                    self.samtools_rmdup[s_name]['n_unique'] = int(match.group(2)) - int(match.group(1))
                    self.samtools_rmdup[s_name]['pct_dups'] = float(match.group(3))*100

        # Filter to strip out ignored sample names
        self.samtools_rmdup = self.ignore_samples(self.samtools_rmdup)

        if len(self.samtools_rmdup) > 0:
            # Write parsed report data to a file
            self.write_data_file(self.samtools_rmdup, 'multiqc_samtools_rmdup')

            # Make a bar plot showing duplicates
            keys = OrderedDict()
            keys['n_unique'] = {'name': 'Non-duplicated reads'}
            keys['n_dups'] = {'name': 'Duplicated reads'}
            pconfig = {
                'id': 'samtools_rmdup_plot',
                'title': 'Samtools rmdup: Duplicate alignments',
                'ylab': 'Number of reads',
                'yDecimals': False
            }
            self.add_section (
                name = 'Duplicates removed',
                anchor = 'samtools-rmdup',
                plot = bargraph.plot(self.samtools_rmdup, keys, pconfig)
            )

            # Add a column to the General Stats table
            # General Stats Table
            stats_headers = OrderedDict()
            stats_headers['pct_dups'] = {
                'title': '% Dups',
                'description': 'Percent of duplicate alignments',
                'min': 0,
                'max': 100,
                'suffix': '%',
                'scale': 'OrRd'
            }
            self.general_stats_addcols(self.samtools_rmdup, stats_headers, 'Samtools rmdup')

        return len(self.samtools_rmdup)
903050
def export(self, input_stats=None):
        
        # threads = []
        input_stats = input_stats or {}

        for e in self._exports:
            logger.debug("Export stats using the %s module" % e)
            thread = threading.Thread(target=self._exports[e].update,
                                      args=(input_stats,))
            # threads.append(thread)
            thread.start()
828444
def parse_status_file(status_file,nas_addr):
    ''' 
    '''
    session_users = {}
    flag1 = False
    flag2 = False
    with open(status_file) as stlines:
        for line in stlines:

            if line.startswith("Common Name"):
                flag1 = True
                continue

            if line.startswith("ROUTING TABLE"):
                flag1 = False
                continue

            if line.startswith("Virtual Address"):
                flag2 = True
                continue

            if line.startswith("GLOBAL STATS"):
                flag2 = False
                continue

            if flag1:
                try:
                    username,realaddr,inbytes,outbytes,_ = line.split(',')
                    realip,realport = realaddr.split(':')
                    session_id = md5(nas_addr + realip + realport).hexdigest()
                    session_users.setdefault(session_id, {}).update(dict(
                        session_id=session_id,
                        username=username,
                        realip=realip,
                        realport=realport,
                        inbytes=inbytes,
                        outbytes=outbytes
                    ))
                except:
                    traceback.print_exc()

            if flag2:
                try:
                    userip,username,realaddr,_ = line.split(',')
                    realip,realport = realaddr.split(':')
                    session_id = md5(nas_addr + realip + realport).hexdigest()
                    session_users.setdefault(session_id, {}).update(dict(
                        session_id=session_id,
                        username=username,
                        realip=realip,
                        realport=realport,
                        userip=userip,
                    ))
                except:
                    traceback.print_exc()

    return session_users
260298
def run(_, data, out_dir=None):
    
    stats_file, idxstats_file = _get_stats_files(data, out_dir)
    samtools = config_utils.get_program("samtools", data["config"])
    bam_file = dd.get_align_bam(data) or dd.get_work_bam(data)
    if not utils.file_exists(stats_file):
        utils.safe_makedir(out_dir)
        with file_transaction(data, stats_file) as tx_out_file:
            cores = dd.get_num_cores(data)
            cmd = "{samtools} stats -@ {cores} {bam_file}"
            cmd += " > {tx_out_file}"
            do.run(cmd.format(**locals()), "samtools stats", data)
    if not utils.file_exists(idxstats_file):
        utils.safe_makedir(out_dir)
        with file_transaction(data, idxstats_file) as tx_out_file:
            cmd = "{samtools} idxstats {bam_file}"
            cmd += " > {tx_out_file}"
            do.run(cmd.format(**locals()), "samtools index stats", data)
    out = {"base": idxstats_file, "secondary": [stats_file]}
    out["metrics"] = _parse_samtools_stats(stats_file)
    return out
1023349
def _comparison_stats_from_merge(in_file, stats, svcaller, data):
    
    truth_stats = {"tp": [], "fn": [], "fp": []}

    samples = ["truth" if x.endswith("-truth") else "eval" for x in vcfutils.get_samples(in_file)]
    with open(in_file) as in_handle:
        for call in (l.rstrip().split("\t") for l in in_handle if not l.startswith("#")):
            supp_vec_str = [x for x in call[7].split(";") if x.startswith("SUPP_VEC=")][0]
            _, supp_vec = supp_vec_str.split("=")
            calls = dict(zip(samples, [int(x) for x in supp_vec]))
            if calls["truth"] and calls["eval"]:
                metric = "tp"
            elif calls["truth"]:
                metric = "fn"
            else:
                metric = "fp"
            truth_stats[metric].append(_summarize_call(call))
    return _to_csv(truth_stats, stats, dd.get_sample_name(data), svcaller)
1359240
def parse_samtools_flagstats(self):
        

        self.samtools_flagstat = dict()
        for f in self.find_log_files('samtools/flagstat'):
            parsed_data = parse_single_report(f['f'])
            if len(parsed_data) > 0:
                if f['s_name'] in self.samtools_flagstat:
                    log.debug("Duplicate sample name found! Overwriting: {}".format(f['s_name']))
                self.add_data_source(f, section='flagstat')
                self.samtools_flagstat[f['s_name']] = parsed_data

        # Filter to strip out ignored sample names
        self.samtools_flagstat = self.ignore_samples(self.samtools_flagstat)

        if len(self.samtools_flagstat) > 0:

            # Write parsed report data to a file (restructure first)
            self.write_data_file(self.samtools_flagstat, 'multiqc_samtools_flagstat')

            # General Stats Table
            flagstats_headers = dict()
            flagstats_headers['mapped_passed'] = {
                'title': '{} Reads Mapped'.format(config.read_count_prefix),
                'description': 'Reads Mapped in the bam file ({})'.format(config.read_count_desc),
                'min': 0,
                'modify': lambda x: x * config.read_count_multiplier,
                'shared_key': 'read_count',
                'placement' : 100.0
            }
            self.general_stats_addcols(self.samtools_flagstat, flagstats_headers, 'Samtools Flagstat')

            # Make dot plot of counts
            keys = OrderedDict()
            reads = {
                'min': 0,
                'modify': lambda x: float(x) * config.read_count_multiplier,
                 'suffix': '{} reads'.format(config.read_count_prefix),
                'decimalPlaces': 2,
                'shared_key': 'read_count'
            }
            keys['flagstat_total']              = dict(reads, title = 'Total Reads' )
            keys['total_passed']                = dict(reads, title = 'Total Passed QC' )
            keys['mapped_passed']               = dict(reads, title = 'Mapped' )

            if any(v.get('secondary_passed') for v in self.samtools_flagstat.values()):
                keys['secondary_passed']        = dict(reads, title = 'Secondary Alignments' )

            if any(v.get('supplementary_passed') for v in self.samtools_flagstat.values()):
                keys['supplementary_passed']    = dict(reads, title = 'Supplementary Alignments' )

            keys['duplicates_passed']           = dict(reads, title = 'Duplicates' )
            keys['paired in sequencing_passed'] = dict(reads, title = 'Paired in Sequencing' )
            keys['properly paired_passed']      = dict(reads, title = 'Properly Paired' )
            keys['with itself and mate mapped_passed'] = \
                                                  dict(reads, title = 'Self and mate mapped',
                                                              description = 'Reads with itself and mate mapped' )
            keys['singletons_passed']           = dict(reads, title = 'Singletons' )
            keys['with mate mapped to a different chr_passed'] = \
                                                  dict(reads, title = 'Mate mapped to diff chr',
                                                              description = 'Mate mapped to different chromosome' )
            keys['with mate mapped to a different chr (mapQ >= 5)_passed'] = \
                                                  dict(reads, title = 'Diff chr (mapQ >= 5)',
                                                              description = 'Mate mapped to different chromosome (mapQ >= 5)' )

            self.add_section (
                name = 'Samtools Flagstat',
                anchor = 'samtools-flagstat',
                description = 'This module parses the output from <code>samtools flagstat</code>. All numbers in millions.',
                plot = beeswarm.plot(self.samtools_flagstat, keys, {'id': 'samtools-flagstat-dp'})
            )

        # Return the number of logs that were found
        return len(self.samtools_flagstat)
21319
def dump_stats(self, pattern):
        
        if not isinstance(pattern, basestring):
            raise TypeError("pattern can only be an instance of type basestring")
        self._call("dumpStats",
                     in_p=[pattern])
523320
def tobam_cl(data, out_file, is_paired=False):
    
    do_dedup = _check_dedup(data)
    umi_consensus = dd.get_umi_consensus(data)
    with file_transaction(data, out_file) as tx_out_file:
        if not do_dedup:
            yield (sam_to_sortbam_cl(data, tx_out_file), tx_out_file)
        elif umi_consensus:
            yield (_sam_to_grouped_umi_cl(data, umi_consensus, tx_out_file), tx_out_file)
        elif is_paired and _need_sr_disc_reads(data) and not _too_many_contigs(dd.get_ref_file(data)):
            sr_file = "%s-sr.bam" % os.path.splitext(out_file)[0]
            disc_file = "%s-disc.bam" % os.path.splitext(out_file)[0]
            with file_transaction(data, sr_file) as tx_sr_file:
                with file_transaction(data, disc_file) as tx_disc_file:
                    yield (samblaster_dedup_sort(data, tx_out_file, tx_sr_file, tx_disc_file),
                           tx_out_file)
        else:
            yield (_biobambam_dedup_sort(data, tx_out_file), tx_out_file)
845600
def collect(self, file=sys.stderr):
        
        cur = gcstats()
        Ncur = len(cur)
        if self.stats is not None and file is not None:
            prev = self.stats
            Nprev = self.ntypes  # may be less than len(prev)

            if Ncur != Nprev:
                print("# Types %d -> %d" % (Nprev, Ncur), file=file)

            Scur, Sprev, first = set(cur), set(prev), True
            for T in Scur - Sprev:  # new types
                if first:
                    print('New Types', file=file)
                    first = False
                print(' ', T, cur[T], file=file)

            first = True
            for T in Sprev - Scur:  # collected types
                if first:
                    print('Cleaned Types', file=file)
                    first = False
                print(' ', T, -prev[T], file=file)

            first = True
            for T in Scur & Sprev:
                if cur[T] == prev[T]:
                    continue
                if first:
                    print('Known Types', file=file)
                    first = False
                print(' ', T, cur[T], 'delta', cur[T] - prev[T], file=file)

        else:  # first call
            print("All Types", file=file)
            for T, C in cur.items():
                print(' ', T, C, file=file)

        self.stats, self.ntypes = cur, len(cur)
273843
def format_stats(self, stats:TensorOrNumList)->None:
        ""
        str_stats = []
        for name,stat in zip(self.names,stats):
            str_stats.append('#na#' if stat is None else str(stat) if isinstance(stat, int) else f'{stat:.6f}')
        if self.add_time: str_stats.append(format_time(time() - self.start_epoch))
        if not self.silent: self.pbar.write(str_stats, table=True)