Add a section for the quality score vs number of observations line plot
198339
def add_quality_score_vs_no_of_observations_section(self):
        

        sample_data = []
        data_labels = []
        for rt_type_name, rt_type in recal_table_type._asdict().items():
            sample_tables = self.gatk_base_recalibrator[rt_type]['quality_quantization_map']
            if len(sample_tables) == 0:
                continue

            sample_data.append({
                sample: {int(x): int(y) for x, y in zip(table['QualityScore'], table['Count'])}
                for sample, table in sample_tables.items()
            })

            sample_y_sums = {
                sample: sum(int(y) for y in table['Count'])
                for sample, table
                in sample_tables.items()
            }

            sample_data.append({
                sample: {
                    int(x): float(y) / sample_y_sums[sample]
                    for x, y in zip(table['QualityScore'], table['Count'])
                }
                for sample, table in sample_tables.items()
            })

            flat_proportions = [float(y) / sample_y_sums[sample]
                                for sample, table in sample_tables.items()
                                for y in table['Count']]
            prop_ymax = max(flat_proportions)
            data_labels.append({'name': "{} Count".format(rt_type_name.capitalize().replace('_', '-')),
                                'ylab': 'Count'})
            data_labels.append({'ymax': prop_ymax,
                                'name': "{} Percent".format(rt_type_name.capitalize().replace('_', '-')),
                                'ylab': 'Percent'})

        plot = linegraph.plot(
            sample_data,
            pconfig={
                'title': "Observed Quality Score Counts",
                'id': 'gatk-base-recalibrator-quality-score-vs-number-of-observations',
                'xlab': 'Observed Quality Score',
                'ylab': 'Count',
                'xDecimals': False,
                'data_labels': data_labels,
            })

        # Reported vs empirical quality scores
        self.add_section(
            name='Observed Quality Scores',
            description=(
                'This plot shows the distribution of base quality scores in each sample before and '
                'after base quality score recalibration (BQSR). Applying BQSR should broaden the '
                'distribution of base quality scores.'
            ),
            helptext=(
                'For more information see '
                '[the Broad\'s description of BQSR]'
                '(https://gatkforums.broadinstitute.org/gatk/discussion/44/base-quality-score-recalibration-bqsr)'
                '.'
            ),
            plot=plot,
        )
446134
def add_readlen_dist_plot(self):
        
        pconfig = {
            'id': 'skewer_read_length_histogram',
            'title': 'Skewer: Read Length Distribution after trimming',
            'xDecimals': False,
            'ylab': '% of Reads',
            'xlab': 'Read Length',
            'xmin': 0,
            'ymin': 0,
            'tt_label': '<b>{point.x}</b>: {point.y:.1f}%',
        }
        self.add_section(
            plot = linegraph.plot(self.skewer_readlen_dist, pconfig)
        )
174112
def _add_section_to_report(self, data):
    

    # Count samples that have errors and/or warnings
    pass_count = error_count = only_warning_count = 0
    for sample_data in data.values():
        if sample_data['file_validation_status'] == 'fail':
            error_count += 1
        elif sample_data['file_validation_status'] == 'warn':
            only_warning_count += 1
        else:
            pass_count += 1

    # Add overview note
    plot_html = []
    note_html = _generate_overview_note(
                        pass_count=pass_count,
                        only_warning_count=only_warning_count,
                        error_count=error_count,
                        total_count=len(data)
                    )
    plot_html.append(note_html)

    # Add the detailed table, but only if we have something to show
    if error_count or only_warning_count:
        table_html = _generate_detailed_table(data)
        plot_html.append(table_html)

    # Finally, add the html to the report as a section
    self.add_section(
            name='SAM/BAM File Validation',
            anchor='picard_validatesamfile',
            description=('This tool reports on the validity of a SAM or BAM '
                         'file relative to the SAM-format specification.'),
            helptext='''
            A detailed table is only shown if errors or warnings are found. Details about the errors and warnings are only shown if a `SUMMARY` report was parsed.

            For more information on the warnings, errors and possible fixes please read [this broadinstitute article](https://software.broadinstitute.org/gatk/documentation/article.php?id=7571).''',
            plot="\n".join(plot_html),
    )
194064
def add_barplot(self):
        
        cats = OrderedDict()
        cats['n_nondups'] = {'name': 'Non-duplicates'}
        cats['n_dups'] = {'name': 'Duplicates'}

        pconfig = {
            'id': 'samblaster_duplicates',
            'title': 'Samblaster: Number of duplicate reads',
            'ylab': 'Number of reads'
        }
        self.add_section( plot = bargraph.plot(self.samblaster_data, cats, pconfig) )
214849
def line(h1: Union[Histogram1D, "HistogramCollection"], ax: Axes, *, errors: bool = False, **kwargs):
    """"""


    show_stats = kwargs.pop("show_stats", False)
    show_values = kwargs.pop("show_values", False)
    density = kwargs.pop("density", False)
    cumulative = kwargs.pop("cumulative", False)
    value_format = kwargs.pop("value_format", None)
    text_kwargs = pop_kwargs_with_prefix("text_", kwargs)
    kwargs["label"] = kwargs.get("label", h1.name)

    data = get_data(h1, cumulative=cumulative, density=density)
    _apply_xy_lims(ax, h1, data, kwargs)
    _add_ticks(ax, h1, kwargs)
    _add_labels(ax, h1, kwargs)

    if errors:
        err_data = get_err_data(h1, cumulative=cumulative, density=density)
        ax.errorbar(h1.bin_centers, data, yerr=err_data, fmt=kwargs.pop(
            "fmt", "-"), ecolor=kwargs.pop("ecolor", "black"), **kwargs)
    else:
        ax.plot(h1.bin_centers, data, **kwargs)

    if show_stats:
        _add_stats_box(h1, ax, stats=show_stats)
    if show_values:
        _add_values(ax, h1, data, value_format=value_format, **text_kwargs)
706485
def plot_accuracy(data, output_dir_path='.', output_filename='accuracy.png',
                  width=10, height=8):
    
    output_path = os.path.join(output_dir_path, output_filename)

    max_val_data = get_epoch_max_val_acc(data)
    max_val_label = round(max_val_data['acc'].values[0], 4)

    # max_val_epoch = max_val_data['epoch'].values[0]
    max_epoch_data = data[data['epoch'] == data['epoch'].max()]

    plot = ggplot(data, aes('epoch', 'acc', color='factor(data)')) + \
        geom_line(size=1, show_legend=False) + \
        geom_vline(aes(xintercept='epoch', color='data'),
                   data=max_val_data, alpha=0.5, show_legend=False) + \
        geom_label(aes('epoch', 'acc'), data=max_val_data,
                   label=max_val_label, nudge_y=-0.02, va='top', label_size=0,
                   show_legend=False) + \
        geom_text(aes('epoch', 'acc', label='data'), data=max_epoch_data,
                  nudge_x=2, ha='center', show_legend=False) + \
        geom_point(aes('epoch', 'acc'), data=max_val_data,
                   show_legend=False) + \
        labs(y='Accuracy', x='Epochs') + \
        theme_bw(base_family='Arial', base_size=15) + \
        scale_color_manual(['#ef8a62', '#67a9cf', "#f7f7f7"])

    plot.save(output_path, width=width, height=height)
28741
def _add_graph_level(graph, level, parent_ids, names, scores, normalized_scores,
                     include_pad):
    """"""
    for i, parent_id in enumerate(parent_ids):
        if not include_pad and names[i] == PAD_TOKEN:
            continue
        new_node = (level, i)
        parent_node = (level - 1, parent_id)
        raw_score = '%.3f' % float(scores[i]) if scores[i] is not None else '-inf'
        norm_score = '%.3f' % float(normalized_scores[i]) if normalized_scores[i] is not None else '-inf'

        graph.add_node(new_node)
        graph.node[new_node]["name"] = names[i]
        graph.node[new_node]["score"] = "[RAW] {}".format(raw_score)
        graph.node[new_node]["norm_score"] = "[NORM] {}".format(norm_score)
        graph.node[new_node]["size"] = 100
        # Add an edge to the parent
        graph.add_edge(parent_node, new_node)
56859
def after_qc_bad_reads_chart(self):
        
        # Specify the order of the different possible categories
        keys = OrderedDict()
        keys['good_reads'] =                     { 'name': 'Good Reads' }
        keys['bad_reads_with_bad_barcode'] =     { 'name': 'Bad Barcode' }
        keys['bad_reads_with_bad_overlap'] =     { 'name': 'Bad Overlap' }
        keys['bad_reads_with_bad_read_length'] = { 'name': 'Bad Read Length' }
        keys['bad_reads_with_low_quality'] =     { 'name': 'Low Quality' }
        keys['bad_reads_with_polyX'] =           { 'name': 'PolyX' }
        keys['bad_reads_with_reads_in_bubble'] = { 'name': 'Reads In Bubble' }
        keys['bad_reads_with_too_many_N'] =      { 'name': 'Too many N' }

        # Config for the plot
        pconfig = {
            'id': 'afterqc_bad_reads_plot',
            'title': 'AfterQC: Filtered Reads',
            'ylab': '# Reads',
            'cpswitch_counts_label': 'Number of Reads',
            'hide_zero_cats': False,
        }
        return bargraph.plot(self.afterqc_data, keys, pconfig)
1122597
def add_significance_indicator(plot, col_a=0, col_b=1, significant=False):
    
    plot_bottom, plot_top = plot.get_ylim()
    # Give the plot a little room for the significance indicator
    line_height = vertical_percent(plot, 0.1)
    # Add some extra spacing below the indicator
    plot_top = plot_top + line_height
    # Add some extra spacing above the indicator
    plot.set_ylim(top=plot_top + line_height * 2)
    color = "black"
    line_top = plot_top + line_height
    plot.plot([col_a, col_a, col_b, col_b], [plot_top, line_top, line_top, plot_top], lw=1.5, color=color)
    indicator = "*" if significant else "ns"
    plot.text((col_a + col_b) * 0.5, line_top, indicator, ha="center", va="bottom", color=color)
400184
def sortmerna_detailed_barplot (self):
        

        # Specify the order of the different possible categories
        keys = OrderedDict()
        metrics = set()
        for sample in self.sortmerna:
            for key in self.sortmerna[sample]:
                if not key in ["total", "rRNA", "non_rRNA"] and not "_pct" in key:
                    metrics.add(key)

        for key in metrics:
            keys[key] = { 'name': key.replace("_count","") }

        # Config for the plot
        pconfig = {
            'id': 'sortmerna-detailed-plot',
            'title': 'SortMeRNA: Hit Counts',
            'ylab': 'Reads'
        }

        self.add_section( plot = bargraph.plot(self.sortmerna, keys, pconfig) )
538394
def precision_curve(self, delta_tau=0.01):
        
        # compute thresholds based on the sorted probabilities
        orig_thresh = self.threshold
        sorted_labels, sorted_probs = self.sorted_values

        scores = []
        taus = []
        tau = 0
        for k in range(len(sorted_labels)):
            # compute new accuracy
            self.threshold = tau
            scores.append(self.precision)
            taus.append(tau)

            # update threshold
            tau = sorted_probs[k]

        # add last datapoint
        tau = 1.0
        self.threshold = tau
        scores.append(1.0)
        taus.append(tau)

        self.threshold = orig_thresh
        return scores, taus
1293510
private void accum_subset(Chunk chks[], Chunk nids, Chunk wrks, int nnids[]) {
      for( int row=0; row<nnids.length; row++ ) { // Over all rows
        int nid = nnids[row];                     // Get Node to decide from
        if( nid >= 0 ) {        // row already predicts perfectly or OOB
          assert !Double.isNaN(wrks.at0(row)); // Already marked as sampled-away
          DHistogram nhs[] = _hcs[nid];
          int sCols[] = _tree.undecided(nid+_leaf)._scoreCols; // Columns to score (null, or a list of selected cols)
          for( int j=0; j<sCols.length; j++) { // For tracked cols
            final int c = sCols[j];
            nhs[c].incr((float)chks[c].at0(row),wrks.at0(row)); // Histogram row/col
          }
        }
      }
    }
455339
def add_dummy_scores(iteratable, score=0):
    """"""
    for seq in iteratable:
        seq.letter_annotations["phred_quality"] = (score,)*len(seq)
        yield seq
868219
def plotter(molecules, ensemble_lookup, options):
    

    try:
        import matplotlib

        matplotlib.use('Agg')
        import matplotlib.pyplot as plt
    except ImportError:
        print("\n Plotting requires matplotlib to be installed\n")
        sys.exit(1)

    for ensemble in ensemble_lookup.keys():

        # create figure
        fig = plt.figure()

        # create the queries subplot, the left subplot
        # create the left hand subplot
        ax1 = fig.add_subplot(121)

        for query in sorted(ensemble_lookup[ensemble]):
            query_list = []
            query_list.append(query)
            score_structure = classification.make_score_structure(molecules, query_list)
            auc_structure = classification.make_auc_structure(score_structure)
            tpf = []
            fpf = []
            for mol in auc_structure:
                fpf.append(mol[4])
                tpf.append(mol[5])

            # add axis-labels and a title
            ax1.set_xlabel('FPF')
            ax1.set_ylabel('TPF')
            title = 'query performance'
            ax1.set_title(title)

            # add plot data and labels for the legend
            lbl = query
            ax1.plot(fpf, tpf, lw=3, label=lbl)

        # get legend handles and labels, then reverse their order
        handles, labels = ax1.get_legend_handles_labels()
        ax1.legend(handles[::-1], labels[::-1])

        # add the legend
        ax1.legend(handles, labels, loc='best')

        # create the ensemble subplot, the right subplot
        score_structure = classification.make_score_structure(molecules, ensemble_lookup[ensemble])
        auc_structure = classification.make_auc_structure(score_structure)
        tpf = []
        fpf = []
        for mol in auc_structure:
            fpf.append(mol[4])
            tpf.append(mol[5])

        # create right hand subplot
        ax2 = fig.add_subplot(122)

        # add axis-labels and a title
        ax2.set_xlabel('FPF')
        ax2.set_ylabel('TPF')
        title = 'ensemble performance'
        ax2.set_title(title)

        # add plot data and a label for the legend
        lbl = 'ensemble'
        ax2.plot(fpf, tpf, lw=3, label=lbl)

        # get legend handles and labels, then reverse their order
        handles, labels = ax2.get_legend_handles_labels()
        ax2.legend(handles[::-1], labels[::-1])

        # add the legend
        ax2.legend(handles, labels, loc='best')

        # save figure
        figurename = options.outname + '_' + ensemble.replace('.csv', '') + '.pdf'
        filename = os.path.join(os.getcwd(), figurename)
        plt.savefig(filename, bbobx='tight', format='pdf')
635749
def fqscreen_simple_plot(self):
        

        # First, sum the different types of alignment counts
        data = OrderedDict()
        cats = OrderedDict()
        for s_name in sorted(self.fq_screen_data):
            data[s_name] = OrderedDict()
            sum_alignments = 0
            for org in self.fq_screen_data[s_name]:
                if org == 'total_reads':
                    continue
                try:
                    data[s_name][org] = self.fq_screen_data[s_name][org]['counts']['one_hit_one_library']
                except KeyError:
                    log.error("No counts found for '{}' ('{}'). Could be malformed or very old FastQ Screen results.".format(org, s_name))
                    continue
                try:
                    data[s_name][org] += self.fq_screen_data[s_name][org]['counts']['multiple_hits_one_library']
                except KeyError:
                    pass
                sum_alignments += data[s_name][org]
                if org not in cats and org != 'No hits':
                    cats[org] = { 'name': org }

            # Calculate hits in multiple genomes
            if 'total_reads' in self.fq_screen_data[s_name]:
                data[s_name]['Multiple Genomes'] = self.fq_screen_data[s_name]['total_reads'] - sum_alignments

        # Strip empty dicts
        for s_name in list(data.keys()):
            if len(data[s_name]) == 0:
                del data[s_name]

        pconfig = {
            'id': 'fastq_screen',
            'title': 'FastQ Screen',
            'cpswitch_c_active': False,
            'hide_zero_cats': False
        }
        cats['Multiple Genomes'] = { 'name': 'Multiple Genomes', 'color': '#820000' }
        cats['No hits'] = { 'name': 'No hits', 'color': '#cccccc' }

        return bargraph.plot(data, cats, pconfig)
263579
def _add_plots_to_output(out, data):
    
    out["plot"] = {}
    diagram_plot = _add_diagram_plot(out, data)
    if diagram_plot:
        out["plot"]["diagram"] = diagram_plot
    scatter = _add_scatter_plot(out, data)
    if scatter:
        out["plot"]["scatter"] = scatter
    scatter_global = _add_global_scatter_plot(out, data)
    if scatter_global:
        out["plot"]["scatter_global"] = scatter_global
    return out
953136
def compare_variant_type_plot(data):
    
    keys = OrderedDict()
    keys['snps'] = {'name': 'SNPs', 'color': '#7cb5ec'}
    keys['indels'] = {'name': 'InDels', 'color': '#90ed7d'}
    keys['multiallelic_snps'] = {'name': 'multi-allelic SNP', 'color': 'orange'}
    keys['complex_indels'] = {'name': 'Complex InDels', 'color': '#8085e9'}

    total_variants = dict()
    known_variants = dict()
    novel_variants = dict()
    for s_name, values in data.items():
        total_variants[s_name] = {
            'snps': values['TOTAL_SNPS'],
            'indels': values['TOTAL_INDELS'],
            'multiallelic_snps': values['TOTAL_MULTIALLELIC_SNPS'],
            'complex_indels': values['TOTAL_COMPLEX_INDELS'],
        }

        known_variants[s_name] = {
            'snps': values['NUM_IN_DB_SNP'],
            'indels': int(values['TOTAL_INDELS'])-int(values['NOVEL_INDELS']),
            'multiallelic_snps': values['NUM_IN_DB_SNP_MULTIALLELIC'],
            'complex_indels': values['NUM_IN_DB_SNP_COMPLEX_INDELS'],
        }

        novel_variants[s_name] = {
            'snps': values['NOVEL_SNPS'],
            'indels': values['NOVEL_INDELS'],
            'multiallelic_snps': int(values['TOTAL_MULTIALLELIC_SNPS'])-int(values['NUM_IN_DB_SNP_MULTIALLELIC']),
            'complex_indels': int(values['TOTAL_COMPLEX_INDELS'])-int(values['NUM_IN_DB_SNP_COMPLEX_INDELS']),
        }

    plot_conf = {
        'id': 'picard_variantCallingMetrics_variant_type',
        'title': 'Picard: Variants Called',
        'ylab': 'Counts of Variants',
        'hide_zero_cats': False,
        'data_labels': [
            {'name': 'Total'},
            {'name': 'Known'},
            {'name': 'Novel'}
        ],
    }
    return bargraph.plot(data=[total_variants, known_variants, novel_variants],
                         cats=[keys, keys, keys],
                         pconfig=plot_conf)