Takes contents from a fastq_data.txt file and parses out required
        statistics and data. Returns a dict with keys 'stats' and 'data'.
        Data is for plotting graphs, stats are for top table.
325870
def parse_fastqc_report(self, file_contents, s_name=None, f=None):
        

        # Make the sample name from the input filename if we find it
        fn_search = re.search(r"Filename\s+(.+)", file_contents)
        if fn_search:
            s_name = self.clean_s_name(fn_search.group(1) , f['root'])

        if s_name in self.fastqc_data.keys():
            log.debug("Duplicate sample name found! Overwriting: {}".format(s_name))
        self.add_data_source(f, s_name)
        self.fastqc_data[s_name] = { 'statuses': dict() }

        # Parse the report
        section = None
        s_headers = None
        self.dup_keys = []
        for l in file_contents.splitlines():
            if l == '>>END_MODULE':
                section = None
                s_headers = None
            elif l.startswith('>>'):
                (section, status) = l[2:].split("\t", 1)
                section = section.lower().replace(' ', '_')
                self.fastqc_data[s_name]['statuses'][section] = status
            elif section is not None:
                if l.startswith('#'):
                    s_headers = l[1:].split("\t")
                    # Special case: Total Deduplicated Percentage header line
                    if s_headers[0] == 'Total Deduplicated Percentage':
                        self.fastqc_data[s_name]['basic_statistics'].append({
                            'measure': 'total_deduplicated_percentage',
                            'value': float(s_headers[1])
                        })
                    else:
                        # Special case: Rename dedup header in old versions of FastQC (v10)
                        if s_headers[1] == 'Relative count':
                            s_headers[1] = 'Percentage of total'
                        s_headers = [s.lower().replace(' ', '_') for s in s_headers]
                        self.fastqc_data[s_name][section] = list()

                elif s_headers is not None:
                    s = l.split("\t")
                    row = dict()
                    for (i, v) in enumerate(s):
                        v.replace('NaN','0')
                        try:
                            v = float(v)
                        except ValueError:
                            pass
                        row[s_headers[i]] = v
                    self.fastqc_data[s_name][section].append(row)
                    # Special case - need to remember order of duplication keys
                    if section == 'sequence_duplication_levels':
                        try:
                            self.dup_keys.append(float(s[0]))
                        except ValueError:
                            self.dup_keys.append(s[0])

        # Tidy up the Basic Stats
        self.fastqc_data[s_name]['basic_statistics'] = {d['measure']: d['value'] for d in self.fastqc_data[s_name]['basic_statistics']}

        # Calculate the average sequence length (Basic Statistics gives a range)
        length_bp = 0
        total_count = 0
        for d in self.fastqc_data[s_name].get('sequence_length_distribution', {}):
            length_bp += d['count'] * self.avg_bp_from_range(d['length'])
            total_count += d['count']
        if total_count > 0:
            self.fastqc_data[s_name]['basic_statistics']['avg_sequence_length'] = length_bp / total_count
856057
def load_data(fpath, **kwargs):
    
    ext = splitext(fpath)[1]
    if ext in ['.pickle', '.cPkl', '.pkl']:
        return load_cPkl(fpath, **kwargs)
    elif ext in ['.json']:
        return load_json(fpath, **kwargs)
    elif ext in ['.hdf5']:
        return load_hdf5(fpath, **kwargs)
    elif ext in ['.txt']:
        return load_text(fpath, **kwargs)
    elif HAS_NUMPY and ext in ['.npz', '.npy']:
        return load_numpy(fpath, **kwargs)
    else:
        assert False, 'unknown ext=%r for fpath=%r' % (ext, fpath)
1349127
def read_stream_stats(self):
        
        stream_stats = OrderedDict()
        for stream in self.streams.values():
            stream_stats[stream] = stream.read_stats()
        return stream_stats
466547
def parse_reports(self):
    

    # Set up vars
    self.read_gc = dict()
    self.read_gc_pct = dict()

    # Go through files and parse data
    for f in self.find_log_files('rseqc/read_gc'):

        if f['f'].startswith('GC%	read_count'):
            gc = list()
            counts = list()
            for l in f['f'].splitlines():
                s = l.split()
                try:
                    gc.append(float(s[0]))
                    counts.append(float(s[1]))
                except:
                    pass
            if len(gc) > 0:
                sorted_gc_keys = sorted(range(len(gc)), key=lambda k: gc[k])
                total = sum(counts)
                if f['s_name'] in self.read_gc:
                    log.debug("Duplicate sample name found! Overwriting: {}".format(f['s_name']))
                self.add_data_source(f, section='read_GC')
                self.read_gc[f['s_name']] = OrderedDict()
                self.read_gc_pct[f['s_name']] = OrderedDict()
                for i in sorted_gc_keys:
                    self.read_gc[f['s_name']][gc[i]] = counts[i]
                    self.read_gc_pct[f['s_name']][gc[i]] = (counts[i]/total)*100

    # Filter to strip out ignored sample names
    self.read_gc = self.ignore_samples(self.read_gc)

    if len(self.read_gc) > 0:

        # Add line graph to section
        pconfig = {
            'id': 'rseqc_read_gc_plot',
            'title': 'RSeQC: Read GC Content',
            'ylab': 'Number of Reads',
            'xlab': "GC content (%)",
            'xmin': 0,
            'xmax': 100,
            'tt_label': "<strong>{point.x}% GC</strong>: {point.y:.2f}",
            'data_labels': [
                {'name': 'Counts', 'ylab': 'Number of Reads'},
                {'name': 'Percentages', 'ylab': 'Percentage of Reads'}
            ]
        }
        self.add_section (
            name = 'Read GC Content',
            anchor = 'rseqc-read_gc',
            description = '<a href="http://rseqc.sourceforge.net/#read-gc-py" target="_blank">read_GC</a>' \
                " calculates a histogram of read GC content.</p>",
            plot = linegraph.plot([self.read_gc, self.read_gc_pct], pconfig)
        )

    # Return number of samples found
    return len(self.read_gc)
1280115
def _get_stats(self, task, start_date):
        

        stats = []
        stats_dir = self._sdir(task.base_dir)
        date = start_date
        end_date = datetime.date.today()
        delta = datetime.timedelta(days=1)

        while date <= end_date:
            date_str = date.strftime('%Y%m%d')
            filename = os.path.join(stats_dir, '{0}.json'.format(date_str))

            if os.path.exists(filename):
                try:
                    # fetch stats content
                    with open(filename, 'r') as file_:
                        data = json.loads(file_.read())

                    # sort descending by time
                    stats.append((date, sorted(data.iteritems(),
                                               key=lambda x: x[1],
                                               reverse=True)))

                except (json.JSONDecodeError, OSError):
                    pass

            date += delta  # next day

        return stats
708880
def _parse_keyvals(self, line_iter):
        
        out = None
        line = None
        for line in line_iter:
            if len(line) == 1 and line[0].upper() == line[0]:
                break
            else:
                # setup output dictionaries, trimming off blank columns
                if out is None:
                    while not line[-1]:
                        line = line[:-1]
                    out = [{} for _ in line[1:]]
                # add blank values if the line is stripped
                while len(line) < len(out) + 1:
                    line.append("")
                for i in range(len(out)):
                    out[i][line[0]] = line[i+1].strip()
                line = None
        return out, line
502613
def adapter_content_plot (self):
        

        data = dict()
        for s_name in self.fastqc_data:
            try:
                for d in self.fastqc_data[s_name]['adapter_content']:
                    pos = self.avg_bp_from_range(d['position'])
                    for r in self.fastqc_data[s_name]['adapter_content']:
                        pos = self.avg_bp_from_range(r['position'])
                        for a in r.keys():
                            k = "{} - {}".format(s_name, a)
                            if a != 'position':
                                try:
                                    data[k][pos] = r[a]
                                except KeyError:
                                    data[k] = {pos: r[a]}
            except KeyError:
                pass
        if len(data) == 0:
            log.debug('adapter_content not found in FastQC reports')
            return None

        # Lots of these datasets will be all zeros.
        # Only take datasets with > 0.1% adapter contamination
        data = {k:d for k,d in data.items() if max(data[k].values()) >= 0.1 }

        pconfig = {
            'id': 'fastqc_adapter_content_plot',
            'title': 'FastQC: Adapter Content',
            'ylab': '% of Sequences',
            'xlab': 'Position (bp)',
            'yCeiling': 100,
            'yMinRange': 5,
            'ymin': 0,
            'xDecimals': False,
            'tt_label': '<b>Base {point.x}</b>: {point.y:.2f}%',
            'hide_empty': True,
            'yPlotBands': [
                {'from': 20, 'to': 100, 'color': '#e6c3c3'},
                {'from': 5, 'to': 20, 'color': '#e6dcc3'},
                {'from': 0, 'to': 5, 'color': '#c3e6c3'},
            ],
        }

        if len(data) > 0:
            plot_html = linegraph.plot(data, pconfig)
        else:
            plot_html = '<div class="alert alert-info">No samples found with any adapter contamination > 0.1%</div>'

        # Note - colours are messy as we've added adapter names here. Not
        # possible to break down pass / warn / fail for each adapter, which
        # could lead to misleading labelling (fails on adapter types with
        # little or no contamination)

        self.add_section (
            name = 'Adapter Content',
            anchor = 'fastqc_adapter_content',
            description = '''The cumulative percentage count of the proportion of your
            library which has seen each of the adapter sequences at each position.''',
            helptext = '''
            Note that only samples with &ge; 0.1% adapter contamination are shown.

            There may be several lines per sample, as one is shown for each adapter
            detected in the file.

            From the [FastQC Help](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/3%20Analysis%20Modules/10%20Adapter%20Content.html):

            _The plot shows a cumulative percentage count of the proportion
            of your library which has seen each of the adapter sequences at each position.
            Once a sequence has been seen in a read it is counted as being present
            right through to the end of the read so the percentages you see will only
            increase as the read length goes on._
            ''',
            plot = plot_html
        )
691805
def parse_fasta(infile, upper=False):
    '''
    
    '''
    try:
        fp = must_open(infile)
    except:
        fp = infile
    # keep header
    fa_iter = (x[1] for x in groupby(fp, lambda row: row[0] == '>'))
    for header in fa_iter:
        header = next(header)
        if header[0] != '>':
            continue
        # drop '>'
        header = header.strip()[1:]
        # stitch the sequence lines together and make into upper case
        seq = "".join(s.strip() for s in next(fa_iter))
        if upper:
            seq = seq.upper()
        yield header, seq
1294184
def process_fasta(fasta, **kwargs):
    """"""
    logging.info("Nanoget: Starting to collect statistics from a fasta file.")
    inputfasta = handle_compressed_input(fasta, file_type="fasta")
    return ut.reduce_memory_usage(pd.DataFrame(
        data=[len(rec) for rec in SeqIO.parse(inputfasta, "fasta")],
        columns=["lengths"]
    ).dropna())
1275601
def stats(self):
        
        return OrderedDict([
            ("Intervals", len(self.array)),
            ("Mean", self.format_time(self.mean or 0)),
            ("Min", self.format_time(self.min or 0)),
            ("Median", self.format_time(self.median or 0)),
            ("Max", self.format_time(self.max or 0)),
            ("St. Dev.", self.format_time(self.stdev or 0)),
            ("Total", self.format_time(self.exectime or 0)),
        ])
1121368
def count(args):
    
    from jcvi.utils.table import loadtable, write_csv

    p = OptionParser(count.__doc__)
    p.add_option("--dir",
                help="Sub-directory where FASTQC was run [default: %default]")
    p.add_option("--human", default=False, action="store_true",
                help="Human friendly numbers [default: %default]")
    p.set_table()
    p.set_outfile()
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    filenames = args
    subdir = opts.dir
    header = "Filename|Total Sequences|Sequence length|Total Bases".split("|")
    rows = []
    human = opts.human
    for f in filenames:
        folder = f.replace(".gz", "").rsplit(".", 1)[0] + "_fastqc"
        if subdir:
            folder = op.join(subdir, folder)
        summaryfile = op.join(folder, "fastqc_data.txt")

        fqcdata = FastQCdata(summaryfile, human=human)
        row = [fqcdata[x] for x in header]
        rows.append(row)

    print(loadtable(header, rows), file=sys.stderr)
    write_csv(header, rows, sep=opts.sep,
              filename=opts.outfile, align=opts.align)
1102138
def fast_stats(data):
    
    
    total_submission = len(data)
    total_submission_best = 0
    total_submission_best_succeeded = 0
        
    for submission in data:
        if "best" in submission and submission["best"]:
            total_submission_best = total_submission_best + 1
            if "result" in submission and submission["result"] == "success":
                total_submission_best_succeeded += 1
        
    statistics = [
        (_("Number of submissions"), total_submission),
        (_("Evaluation submissions (Total)"), total_submission_best),
        (_("Evaluation submissions (Succeeded)"), total_submission_best_succeeded),
        (_("Evaluation submissions (Failed)"), total_submission_best - total_submission_best_succeeded),
        # add here new common statistics
        ]
    
    return statistics
547969
def statstable(args):
    
    p = OptionParser(statstable.__doc__)
    opts, args = p.parse_args(args)

    if len(args) < 1:
        sys.exit(not p.print_help())

    gff_files = args
    for metric in metrics:
        logging.debug("Parsing files in `{0}`..".format(metric))

        table = {}
        for x in gff_files:
            pf = op.basename(x).split(".")[0]
            numberfile = op.join(metric, pf + ".txt")
            ar = [int(x.strip()) for x in open(numberfile)]
            sum = SummaryStats(ar).todict().items()
            keys, vals = zip(*sum)
            keys = [(pf, x) for x in keys]
            table.update(dict(zip(keys, vals)))

        print(tabulate(table), file=sys.stderr)
1141350
def main():
    '''
    
    '''
    args = get_args()
    try:
        utils.make_output_dir(args.outdir)
        utils.init_logs(args)
        args.format = nanoplotter.check_valid_format(args.format)
        settings = vars(args)
        settings["path"] = path.join(args.outdir, args.prefix)
        sources = {
            "fastq": args.fastq,
            "bam": args.bam,
            "cram": args.cram,
            "fastq_rich": args.fastq_rich,
            "fastq_minimal": args.fastq_minimal,
            "summary": args.summary,
            "fasta": args.fasta,
            "ubam": args.ubam,
        }

        if args.pickle:
            datadf = pickle.load(open(args.pickle, 'rb'))
        else:
            datadf = get_input(
                source=[n for n, s in sources.items() if s][0],
                files=[f for f in sources.values() if f][0],
                threads=args.threads,
                readtype=args.readtype,
                combine="simple",
                barcoded=args.barcoded)
        if args.store:
            pickle.dump(
                obj=datadf,
                file=open(settings["path"] + "NanoPlot-data.pickle", 'wb'))
        if args.raw:
            datadf.to_csv(settings["path"] + "NanoPlot-data.tsv.gz",
                          sep="\t",
                          index=False,
                          compression="gzip")

        settings["statsfile"] = [make_stats(datadf, settings, suffix="")]
        datadf, settings = filter_and_transform_data(datadf, settings)
        if settings["filtered"]:  # Bool set when filter was applied in filter_and_transform_data()
            settings["statsfile"].append(make_stats(datadf, settings, suffix="_post_filtering"))

        if args.barcoded:
            barcodes = list(datadf["barcode"].unique())
            plots = []
            for barc in barcodes:
                logging.info("Processing {}".format(barc))
                settings["path"] = path.join(args.outdir, args.prefix + barc + "_")
                dfbarc = datadf[datadf["barcode"] == barc]
                if len(dfbarc) > 5:
                    settings["title"] = barc
                    plots.extend(
                        make_plots(dfbarc, settings)
                    )
                else:
                    sys.stderr.write("Found barcode {} less than 5x, ignoring...\n".format(barc))
                    logging.info("Found barcode {} less than 5 times, ignoring".format(barc))
        else:
            plots = make_plots(datadf, settings)
        make_report(plots, settings)
        logging.info("Finished!")
    except Exception as e:
        logging.error(e, exc_info=True)
        print("\n\n\nIf you read this then NanoPlot has crashed :-(")
        print("Please try updating NanoPlot and see if that helps...\n")
        print("If not, please report this issue at https://github.com/wdecoster/NanoPlot/issues")
        print("If you could include the log file that would be really helpful.")
        print("Thanks!\n\n\n")
        raise
1120682
def stats_to_list(stats_dict, localize=pytz):
    
    result = stats_dict['GetSendStatisticsResponse']['GetSendStatisticsResult']
    # Make a copy, so we don't change the original stats_dict.
    result = copy.deepcopy(result)
    datapoints = []
    if localize:
        current_tz = localize.timezone(settings.TIME_ZONE)
    else:
        current_tz = None
    for dp in result['SendDataPoints']:
        if current_tz:
            utc_dt = datetime.strptime(dp['Timestamp'], '%Y-%m-%dT%H:%M:%SZ')
            utc_dt = localize.utc.localize(utc_dt)
            dp['Timestamp'] = current_tz.normalize(
                utc_dt.astimezone(current_tz))
        datapoints.append(dp)

    datapoints.sort(key=lambda x: x['Timestamp'])

    return datapoints
886307
def seq_length_dist_plot (self):
        

        data = dict()
        seq_lengths = set()
        multiple_lenths = False
        for s_name in self.fastqc_data:
            try:
                data[s_name] = {self.avg_bp_from_range(d['length']): d['count'] for d in self.fastqc_data[s_name]['sequence_length_distribution']}
                seq_lengths.update(data[s_name].keys())
                if len(set(data[s_name].keys())) > 1:
                    multiple_lenths = True
            except KeyError:
                pass
        if len(data) == 0:
            log.debug('sequence_length_distribution not found in FastQC reports')
            return None

        if not multiple_lenths:
            lengths = 'bp , '.join([str(l) for l in list(seq_lengths)])
            desc = 'All samples have sequences of a single length ({}bp).'.format(lengths)
            if len(seq_lengths) > 1:
                desc += ' See the <a href="#general_stats">General Statistics Table</a>.'
            self.add_section (
                name = 'Sequence Length Distribution',
                anchor = 'fastqc_sequence_length_distribution',
                description = '<div class="alert alert-info">{}</div>'.format(desc)
            )
        else:
            pconfig = {
                'id': 'fastqc_sequence_length_distribution_plot',
                'title': 'FastQC: Sequence Length Distribution',
                'ylab': 'Read Count',
                'xlab': 'Sequence Length (bp)',
                'ymin': 0,
                'yMinTickInterval': 0.1,
                'xDecimals': False,
                'colors': self.get_status_cols('sequence_length_distribution'),
                'tt_label': '<b>{point.x} bp</b>: {point.y}',
            }
            self.add_section (
                name = 'Sequence Length Distribution',
                anchor = 'fastqc_sequence_length_distribution',
                description = '''The distribution of fragment sizes (read lengths) found.
                    See the [FastQC help](http://www.bioinformatics.babraham.ac.uk/projects/fastqc/Help/3%20Analysis%20Modules/7%20Sequence%20Length%20Distribution.html)''',
                plot = linegraph.plot(data, pconfig)
            )
690128
def interop_parse(self):
        
        # Parse the files and load the data
        try:
            run_metrics = py_interop_run_metrics.run_metrics()
            valid_to_load = py_interop_run.uchar_vector(py_interop_run.MetricCount, 0)
            py_interop_run_metrics.list_summary_metrics_to_load(valid_to_load)
            run_metrics.read(self.path, valid_to_load)
            summary = py_interop_summary.run_summary()
            py_interop_summary.summarize_run_metrics(run_metrics, summary)
            # PhiX error rate for run over all "usable cycles"
            errorrate = summary.total_summary().error_rate()
            # Percent aligned PhiX
            pctaligned = summary.total_summary().percent_aligned()
            # Add the error rate and the percent of reads that align to PhiX to the metadata object
            for sample in self.metadata:
                sample.run.error_rate = '{:.2f}'.format(errorrate)
                sample.run.phix_aligned = '{:.2f}'.format(pctaligned)
        except:
            for sample in self.metadata:
                sample.run.error_rate = 'ND'
                sample.run.phix_aligned = 'ND'