Mutation Score Indicator (MSI)
180334
public function getMutationScoreIndicator(): float
    {
        $detectionRateAll = 0;
        $defeatedTotal = $this->killedCount + $this->timedOutCount + $this->errorCount;

        if ($this->totalMutantsCount) {
            $detectionRateAll = floor(100 * $defeatedTotal / $this->totalMutantsCount);
        }

        return $detectionRateAll;
    }
523194
def get_atlas_per_gene_mutation_df(self, gene_id):
        
        # TODO: also count: number of unique mutations (have to consider position, amino acid change)
        # TODO: keep track of strain with most mutations, least mutations
        # TODO: keep track of strains that conserve the length of the protein, others that extend or truncate it
        # need statistical test for that too (how long is "extended"/"truncated"?)
        # TODO: number of strains with at least 1 mutations
        # TODO: number of strains with <5% mutated, 5-10%, etc

        g = self.reference_gempro.genes.get_by_id(gene_id)

        single, fingerprint = g.protein.sequence_mutation_summary(alignment_type='seqalign')

        structure_type_suffix = 'NA'
        appender = []

        for k, strains in single.items():
            # Mutations in the strain
            to_append = {}
            orig_res = k[0]
            resnum = int(k[1])
            mutated_res = k[2]
            num_strains_mutated = len(strains)
            strain_ids = [str(x.split(g.id + '_')[1]) for x in strains]
            to_append['ref_residue'] = orig_res
            to_append['ref_resnum'] = resnum
            to_append['strain_residue'] = mutated_res
            to_append['num_strains_mutated'] = num_strains_mutated
            to_append['strains_mutated'] = ';'.join(strain_ids)
            to_append['at_disulfide_bridge'] = False

            # Residue properties
            origres_props = ssbio.protein.sequence.properties.residues.residue_biochemical_definition(orig_res)
            mutres_props = ssbio.protein.sequence.properties.residues.residue_biochemical_definition(mutated_res)
            to_append['ref_residue_prop'] = origres_props
            to_append['strain_residue_prop'] = mutres_props

            # Grantham score - score a mutation based on biochemical properties
            grantham_s, grantham_txt = ssbio.protein.sequence.properties.residues.grantham_score(orig_res, mutated_res)
            to_append['grantham_score'] = grantham_s
            to_append['grantham_annotation'] = grantham_txt

            # Get all per residue annotations - predicted from sequence and calculated from structure
            to_append.update(g.protein.get_residue_annotations(seq_resnum=resnum, use_representatives=True))

            # Check structure type
            if g.protein.representative_structure:
                if g.protein.representative_structure.is_experimental:
                    to_append['structure_type'] = 'EXP'
                else:
                    to_append['structure_type'] = 'HOM'

                # At disulfide bond?
                repchain = g.protein.representative_chain
                repchain_annotations = g.protein.representative_structure.chains.get_by_id(repchain).seq_record.annotations
                if 'SSBOND-biopython' in repchain_annotations:
                    structure_resnum = g.protein.map_seqprop_resnums_to_structprop_resnums(resnums=resnum,
                                                                                           use_representatives=True)
                    if resnum in structure_resnum:
                        ssbonds = repchain_annotations['SSBOND-biopython']
                        ssbonds_res = []
                        for x in ssbonds:
                            ssbonds_res.append(x[0])
                            ssbonds_res.append(x[1])

                        if structure_resnum in ssbonds_res:
                            to_append['at_disulfide_bridge'] = True

            appender.append(to_append)

        if not appender:
            return pd.DataFrame()

        cols = ['ref_residue', 'ref_resnum', 'strain_residue', 'num_strains_mutated', 'strains_mutated',
                'ref_residue_prop', 'strain_residue_prop', 'grantham_score', 'grantham_annotation',
                'at_disulfide_bridge',
                'seq_SS-sspro', 'seq_SS-sspro8', 'seq_RSA-accpro', 'seq_RSA-accpro20', 'seq_TM-tmhmm',
                'struct_SS-dssp', 'struct_RSA-dssp', 'struct_ASA-dssp',
                'struct_CA_DEPTH-msms', 'struct_RES_DEPTH-msms',
                'struct_PHI-dssp', 'struct_PSI-dssp',
                'struct_resnum', 'struct_residue'
                'strains_mutated']

        df_gene_summary = pd.DataFrame.from_records(appender, columns=cols)

        # Drop columns that don't have anything in them
        df_gene_summary.dropna(axis=1, how='all', inplace=True)

        df_gene_summary.sort_values(by='ref_resnum', inplace=True)
        df_gene_summary = df_gene_summary.set_index('ref_resnum')
        return df_gene_summary
1027887
private void tinyGPEvolve()
   {
      int iPopSize = this.manager.getPopulationSize();

      double sum_rate = manager.getCrossoverRate()+manager.getMacroMutationRate()+manager.getMicroMutationRate()+manager.getReproductionRate();
      double crossover_disk = manager.getCrossoverRate() / sum_rate;
      double micro_mutation_disk = (manager.getCrossoverRate() + manager.getMicroMutationRate()) / sum_rate;
      double macro_mutation_disk = (manager.getCrossoverRate() + manager.getMicroMutationRate() + manager.getMacroMutationRate()) / sum_rate;

      RandEngine randEngine = manager.getRandEngine();

      bestSolutionInCurrentGeneration = globalBestSolution.orElse(null);
      for (int offspring_index = 0; offspring_index < iPopSize; offspring_index += 1)
      {
         double r = randEngine.uniform();
         List<Solution> children = new ArrayList<>();

         List<Solution> bad_parents = new ArrayList<>();

         TournamentSelectionResult<Solution> tournament = TournamentSelection.select(solutions, randEngine);
         TupleTwo<Solution, Solution> tournament_winners = tournament.getWinners();
         TupleTwo<Solution, Solution> tournament_losers = tournament.getLosers();

         bad_parents.add(tournament_losers._1());
         bad_parents.add(tournament_losers._2());

         if (r <= crossover_disk) // do crossover
         {
            Solution child1 = tournament_winners._1().makeCopy();
            Solution child2 = tournament_winners._2().makeCopy();

            Crossover.apply(child1, child2, manager);

            children.add(child1);
            children.add(child2);
         }
         else if (r <= micro_mutation_disk) // do point mutation
         {
            Solution child = tournament_winners._1().makeCopy();
            MicroMutation.apply(child, manager);
            children.add(child);
         }
         else if (r <= macro_mutation_disk) // do subtree mutation
         {
            Solution child = tournament_winners._1().makeCopy();
            MacroMutation.apply(child, manager);
            children.add(child);
         }
         else // do reproduction
         {
            Solution child = tournament_winners._1().makeCopy();
            children.add(child);
         }


         boolean successfully_replaced = false;
         for(int i=0; i < children.size(); ++i)
         {
            Solution child = children.get(i);

            evaluate(child);

            if (bestSolutionInCurrentGeneration == null || child.isBetterThan(bestSolutionInCurrentGeneration))
            {
               bestSolutionInCurrentGeneration = child;
            }

            for (Solution bad_parent : bad_parents)
            {
               if (child.isBetterThan(bad_parent))
               {
                  successfully_replaced = true;
                  solutions.set(solutions.indexOf(bad_parent), child);
                  break;
               }
            }

            if (successfully_replaced)
            {
               break;
            }
         }

      }

      updateGlobal(bestSolutionInCurrentGeneration);
   }
57126
def legacy_reporter(self):
        
        from collections import OrderedDict
        logging.info('Creating legacy summary report')
        row = ''
        # Create a dictionary of tuples to be printed in the final report
        for sample in self.metadata:
            data = OrderedDict([
                ('SampleName', sample.name),
                ('N50', str(sample.quality_features_polished.n50)),
                ('NumContigs', str(sample.quality_features_polished.num_contigs)),
                ('TotalLength', str(sample.quality_features_polished.genome_length)),
                ('MeanInsertSize', sample.mapping.MeanInsertSize),
                ('AverageCoverageDepth', sample.mapping.MeanCoveragedata.split("X")[0]),
                ('ReferenceGenome', sample.mash.closestrefseq),
                ('RefGenomeAlleleMatches', '-'),
                ('16sPhylogeny', sample.sixteens_full.genus),
                ('rMLSTsequenceType', sample.rmlst.sequencetype),
                ('MLSTsequencetype', sample.mlst.sequencetype),
                ('MLSTmatches', str(sample.mlst.matchestosequencetype)),
                ('coreGenome', GenObject.returnattr(sample.coregenome, 'coreresults').rstrip(',')),
                ('SeroType', '{oset}:{hset}'
                    .format(oset=';'.join(sample.serosippr.o_set),
                            hset=';'.join(sample.serosippr.h_set))),
                ('geneSeekrProfile', ';'.join(result for result, pid in sorted(sample.genesippr.results.items()))),
                ('vtyperProfile', ';'.join(sorted(sample.legacy_vtyper.toxinprofile))),
                ('percentGC', str(sample.quality_features_polished.gc)),
                ('TotalPredictedGenes', str(sample.prodigal.predictedgenestotal)),
                ('predictedgenesover3000bp', str(sample.prodigal.predictedgenesover3000bp)),
                ('predictedgenesover1000bp', str(sample.prodigal.predictedgenesover1000bp)),
                ('predictedgenesover500bp', str(sample.prodigal.predictedgenesover500bp)),
                ('predictedgenesunder500bp', str(sample.prodigal.predictedgenesunder500bp)),
                ('SequencingDate', sample.run.Date),
                ('Investigator', sample.run.InvestigatorName),
                ('TotalClustersinRun', str(sample.run.TotalClustersinRun)),
                ('NumberofClustersPF', str(sample.run.NumberofClustersPF)),
                ('PercentOfClusters', str(sample.run.PercentOfClusters)),
                ('LengthofForwardRead', str(sample.run.forwardlength)),
                ('LengthofReverseRead', str(sample.run.reverselength)),
                ('Project', str(sample.run.SampleProject)),
                ('PipelineVersion', self.commit)
            ])

            if not row:
                row += ','.join([key for key, value in data.items()])
            row += '\n'
            row += ','.join([value for key, value in data.items()])
        cleanrow = row.replace('NA', '').replace(',-,', ',,')
        with open(os.path.join(self.reportpath, 'legacy_combinedMetadata.csv'), 'w') as metadatareport:
            metadatareport.write(cleanrow)
918847
static List<ExperimentAlgorithm<DoubleSolution, List<DoubleSolution>>> configureAlgorithmList(
          List<ExperimentProblem<DoubleSolution>> problemList) {
    List<ExperimentAlgorithm<DoubleSolution, List<DoubleSolution>>> algorithms = new ArrayList<>();

    for (int run = 0; run < INDEPENDENT_RUNS; run++) {

      for (int i = 0; i < problemList.size(); i++) {
        double mutationProbability = 1.0 / problemList.get(i).getProblem().getNumberOfVariables();
        double mutationDistributionIndex = 20.0;
        Algorithm<List<DoubleSolution>> algorithm = new SMPSOBuilder(
                (DoubleProblem) problemList.get(i).getProblem(),
                new CrowdingDistanceArchive<DoubleSolution>(100))
                .setMutation(new PolynomialMutation(mutationProbability, mutationDistributionIndex))
                .setMaxIterations(250)
                .setSwarmSize(100)
                .setSolutionListEvaluator(new SequentialSolutionListEvaluator<DoubleSolution>())
                .build();
        algorithms.add(new ExperimentAlgorithm<>(algorithm, problemList.get(i), run));
      }

      for (int i = 0; i < problemList.size(); i++) {
        Algorithm<List<DoubleSolution>> algorithm = new NSGAIIBuilder<DoubleSolution>(
                problemList.get(i).getProblem(),
                new SBXCrossover(1.0, 20.0),
                new PolynomialMutation(1.0 / problemList.get(i).getProblem().getNumberOfVariables(),
                        20.0),
                100)
                .build();
        algorithms.add(new ExperimentAlgorithm<>(algorithm, problemList.get(i), run));
      }

      for (int i = 0; i < problemList.size(); i++) {
        Algorithm<List<DoubleSolution>> algorithm = new MOEADBuilder(problemList.get(i).getProblem(), MOEADBuilder.Variant.MOEAD)
                .setCrossover(new DifferentialEvolutionCrossover(1.0, 0.5, "rand/1/bin"))
                .setMutation(new PolynomialMutation(1.0 / problemList.get(i).getProblem().getNumberOfVariables(),
                        20.0))
                .setMaxEvaluations(25000)
                .setPopulationSize(100)
                .setResultPopulationSize(100)
                .setNeighborhoodSelectionProbability(0.9)
                .setMaximumNumberOfReplacedSolutions(2)
                .setNeighborSize(20)
                .setFunctionType(AbstractMOEAD.FunctionType.TCHE)
                .build();
        algorithms.add(new ExperimentAlgorithm<>(algorithm, problemList.get(i), run));
      }
    }
    return algorithms;
  }
980216
def get_qc_tools(data):
    
    if dd.get_algorithm_qc(data):
        return dd.get_algorithm_qc(data)
    analysis = data["analysis"].lower()
    to_run = []
    if tz.get_in(["config", "algorithm", "kraken"], data):
        to_run.append("kraken")
    if "fastqc" not in dd.get_tools_off(data):
        to_run.append("fastqc")
    if any([tool in dd.get_tools_on(data)
            for tool in ["qualimap", "qualimap_full"]]):
        to_run.append("qualimap")
    if analysis.startswith("rna-seq") or analysis == "smallrna-seq":
        if "qualimap" not in dd.get_tools_off(data):
            if gtf.is_qualimap_compatible(dd.get_gtf_file(data)):
                to_run.append("qualimap_rnaseq")
            else:
                logger.debug("GTF not compatible with Qualimap, skipping.")
    if analysis.startswith("chip-seq"):
        to_run.append("chipqc")
    if analysis.startswith("smallrna-seq"):
        to_run.append("small-rna")
        to_run.append("atropos")
    if "coverage_qc" not in dd.get_tools_off(data):
        to_run.append("samtools")
    if dd.has_variantcalls(data):
        if "coverage_qc" not in dd.get_tools_off(data):
            to_run += ["coverage", "picard"]
        to_run += ["qsignature", "variants"]
        if vcfanno.is_human(data):
            to_run += ["contamination", "peddy"]
        if vcfutils.get_paired_phenotype(data):
            to_run += ["viral"]
        if damage.should_filter([data]):
            to_run += ["damage"]
    if dd.get_umi_consensus(data):
        to_run += ["umi"]
    if tz.get_in(["config", "algorithm", "preseq"], data):
        to_run.append("preseq")
    to_run = [tool for tool in to_run if tool not in dd.get_tools_off(data)]
    to_run.sort()
    return to_run
1125416
def mstmap(args):
    
    from jcvi.assembly.geneticmap import MSTMatrix

    p = OptionParser(mstmap.__doc__)
    p.add_option("--population_type", default="RIL6",
                 help="Type of population, possible values are DH and RILd")
    p.add_option("--missing_threshold", default=.5,
                 help="Missing threshold, .25 excludes any marker with >25% missing")
    p.set_outfile()
    opts, args = p.parse_args(args)

    if len(args) != 1:
        sys.exit(not p.print_help())

    lmd, = args
    fp = open(lmd)
    next(fp)  # Header
    table = {"0": "-", "1": "A", "2": "B", "3": "X"}
    mh = ["locus_name"] + fp.next().split()[4:]
    genotypes = []
    for row in fp:
        atoms = row.split()
        chr, pos, ref, alt = atoms[:4]
        locus_name = ".".join((chr, pos))
        codes = [table[x] for x in atoms[4:]]
        genotypes.append([locus_name] + codes)

    mm = MSTMatrix(genotypes, mh, opts.population_type, opts.missing_threshold)
    mm.write(opts.outfile, header=True)
321553
def _variant_sv(checkpoints):
    
    if not checkpoints.get("sv"):
        return [], []
    sv = [s("detect_sv", "batch-single",
            [["sv_batch_rec"]],
            [cwlout("sv_rec", "record",
                    fields=[cwlout(["sv", "variantcaller"], ["string", "null"]),
                            cwlout(["sv", "vrn_file"], ["File", "null"], [".tbi"]),
                            cwlout(["sv", "supplemental"], {"type": "array", "items": ["File"]}),
                            cwlout(["svvalidate", "summary"], ["File", "null"]),
                            cwlout("inherit", exclude=[["align_bam"], ["work_bam_plus"],
                                                       ["reference", "snpeff"]])])],
            "bcbio-vc", ["bedtools", "cnvkit", "delly", "duphold", "extract-sv-reads", "gsort",
                         "lumpy-sv;env=python2", "manta;env=python2", "break-point-inspector", "mosdepth", "samtools",
                         "smoove;env=python2", "pysam>=0.13.0",
                         "seq2c", "simple_sv_annotation;env=python2", "survivor", "svtools;env=python2",
                         "svtyper;env=python2",
                         "r=3.5.1", "r-base", "xorg-libxt", "vawk;env=python2"],
            disk={"files": 2.0})]
    sv_batch_inputs = [["analysis"], ["genome_build"],
                       ["work_bam_plus", "disc"], ["work_bam_plus", "sr"],
                       ["config", "algorithm", "background", "cnv_reference"],
                       ["config", "algorithm", "tools_on"],
                       ["config", "algorithm", "tools_off"],
                       ["config", "algorithm", "svprioritize"],
                       ["config", "algorithm", "svvalidate"], ["regions", "sample_callable"],
                       ["genome_resources", "variation", "gc_profile"],
                       ["genome_resources", "variation", "germline_het_pon"],
                       ["genome_resources", "aliases", "snpeff"], ["reference", "snpeff", "genome_build"],
                       ["sv_coverage_rec"]]
    if checkpoints.get("vc"):
        sv_batch_inputs.append(["variants", "samples"])
    steps = [s("calculate_sv_bins", "multi-combined",
               [["align_bam"], ["reference", "fasta", "base"],
                ["metadata", "batch"], ["metadata", "phenotype"],
                ["config", "algorithm", "background", "cnv_reference"],
                ["config", "algorithm", "callable_regions"],
                ["config", "algorithm", "coverage_interval"],
                ["config", "algorithm", "exclude_regions"],
                ["config", "algorithm", "sv_regions"],
                ["config", "algorithm", "variant_regions"],
                ["config", "algorithm", "variant_regions_merged"],
                ["config", "algorithm", "seq2c_bed_ready"],
                ["config", "algorithm", "svcaller"],
                ["depth", "variant_regions", "regions"],
                ["genome_resources", "variation", "lcr"], ["genome_resources", "variation", "polyx"],
                ["genome_resources", "variation", "encode_blacklist"],
                ["genome_resources", "rnaseq", "gene_bed"]],
               [cwlout("sv_bin_rec", "record",
                       fields=[cwlout(["regions", "bins", "target"], ["File", "null"]),
                               cwlout(["regions", "bins", "antitarget"], ["File", "null"]),
                               cwlout(["regions", "bins", "gcannotated"], ["File", "null"]),
                               cwlout(["regions", "bins", "group"], ["string", "null"]),
                               cwlout("inherit")])],
               "bcbio-vc", ["bedtools", "cnvkit"],
               disk={"files": 1.5}, cores=1),
             s("calculate_sv_coverage", "multi-parallel",
               [["sv_bin_rec"]],
               [cwlout("sv_rawcoverage_rec", "record",
                       fields=[cwlout(["depth", "bins", "target"], ["File", "null"]),
                               cwlout(["depth", "bins", "antitarget"], ["File", "null"]),
                               cwlout(["depth", "bins", "seq2c"], ["File", "null"]),
                               cwlout("inherit")])],
               "bcbio-vc", ["mosdepth", "cnvkit", "seq2c"],
               disk={"files": 1.5}),
             s("normalize_sv_coverage", "multi-combined",
               [["sv_rawcoverage_rec"]],
               [cwlout("sv_coverage_rec", "record",
                       fields=[cwlout(["depth", "bins", "normalized"], ["File", "null"]),
                               cwlout(["depth", "bins", "background"], ["File", "null"]),
                               cwlout("inherit")])],
               "bcbio-vc", ["cnvkit"],
               disk={"files": 1.5}),
             s("batch_for_sv", "multi-batch", sv_batch_inputs,
               [cwlout("sv_batch_rec", "record")],
               "bcbio-vc",
               unlist=[["config", "algorithm", "svcaller"]]),
             w("svcall", "multi-parallel", sv, []),
             s("summarize_sv", "multi-combined",
               [["sv_rec"]],
               [cwlout(["sv", "calls"], {"type": "array", "items": ["File", "null"]}),
                cwlout(["sv", "supplemental"], {"type": "array", "items": ["File"]}),
                cwlout(["sv", "prioritize", "tsv"], {"type": "array", "items": ["File", "null"]}),
                cwlout(["sv", "prioritize", "raw"], {"type": "array", "items": ["File", "null"]}),
                cwlout(["svvalidate", "grading_summary"], ["File", "null"]),
                cwlout(["svvalidate", "grading_plots"], {"type": "array", "items": ["File", "null"]})],
               "bcbio-vc", ["bcbio-prioritize"], disk={"files": 1.0}, cores=1)]
    final_outputs = [["sv", "calls"], ["svvalidate", "grading_summary"], ["sv", "prioritize", "tsv"],
                     ["sv", "prioritize", "raw"], ["sv", "supplemental"]]
    return steps, final_outputs
1073366
private String formatDnaAllele() {

//        We are not considering this case since would mean there's no variant
//        if self.mutation_type == '=':
//            #No change.
//            #example:
//        101 A =
//        return self.ref_allele + '='

        if (">".equals(mutationType)) {
            // SNP.
            // example:
            // 101 A > C
            return reference + '>' + alternate;
        } else if ("delins".equals(mutationType)) {
            // Indel.
            // example:
            // 112_117d elAGGTCAinsTG, 112_117d elinsTG
            return "del" + reference + "ins" + alternate;
        } else if ("del".equals(mutationType)) {
            // Delete, duplication.
            // example:
            // 1000_1003d elATG, 1000_1003d upATG
            return mutationType + reference;
        } else if ("dup".equals(mutationType)) {
            // Insertion normalized as duplication
            // example:
            // "ENST00000382869.3:c.1735+32dupA"
            return mutationType + alternate;
        } else if ("ins".equals(mutationType)) {
            // Insert.
            // example:
            // 1000_1001 insATG
            return mutationType + alternate;
        } else {
            throw new AssertionError("unknown mutation type: '" + mutationType + "'");
        }
    }
1370284
def report(self, align_bam, ref_file, is_paired, bait_file, target_file,
               variant_region_file, config):
        
        dup_metrics = self._get_current_dup_metrics(align_bam)
        align_metrics = self._collect_align_metrics(align_bam, ref_file)
        # Prefer the GC metrics in FastQC instead of Picard
        # gc_graph, gc_metrics = self._gc_bias(align_bam, ref_file)
        gc_graph = None
        insert_graph, insert_metrics, hybrid_metrics = (None, None, None)
        if is_paired:
            insert_graph, insert_metrics = self._insert_sizes(align_bam)
        if bait_file and target_file:
            assert os.path.exists(bait_file), (bait_file, "does not exist!")
            assert os.path.exists(target_file), (target_file, "does not exist!")
            hybrid_metrics = self._hybrid_select_metrics(align_bam,
                                                         bait_file, target_file)
        elif (variant_region_file and
              config["algorithm"].get("coverage_interval", "").lower() in ["exome"]):
            assert os.path.exists(variant_region_file), (variant_region_file, "does not exist")
            hybrid_metrics = self._hybrid_select_metrics(
                align_bam, variant_region_file, variant_region_file)

        vrn_vals = self._variant_eval_metrics(align_bam)
        summary_info = self._parser.get_summary_metrics(align_metrics,
                dup_metrics, insert_metrics, hybrid_metrics,
                vrn_vals)
        graphs = []
        if gc_graph and os.path.exists(gc_graph):
            graphs.append((gc_graph, "Distribution of GC content across reads"))
        if insert_graph and os.path.exists(insert_graph):
            graphs.append((insert_graph, "Distribution of paired end insert sizes"))
        return summary_info, graphs
1158840
def generate_bug_changes(self, startday, endday, alt_startday, alt_endday):
        """"""

        bug_stats, bug_ids = self.get_bug_stats(startday, endday)
        alt_date_bug_totals = self.get_alt_date_bug_totals(alt_startday, alt_endday, bug_ids)
        test_run_count = self.get_test_runs(startday, endday)

        # if fetch_bug_details fails, None is returned
        bug_info = self.fetch_all_bug_details(bug_ids)

        all_bug_changes = []
        template = Template(self.open_file('comment.template', False))

        if self.weekly_mode:
            top_bugs = [bug[0] for bug in sorted(bug_stats.items(), key=lambda x: x[1]['total'],
                        reverse=True)][:50]

        for bug_id, counts in bug_stats.items():
            change_priority = None
            change_whiteboard = None
            priority = 0
            rank = top_bugs.index(bug_id)+1 if self.weekly_mode and bug_id in top_bugs else None

            if bug_info and bug_id in bug_info:
                if self.weekly_mode:
                    priority = self.assign_priority(counts)
                    if priority == 2:
                        change_priority, change_whiteboard = self.check_needswork_owner(bug_info[bug_id])

                    # change [stockwell needswork] to [stockwell unknown] when failures drop below 20 failures/week
                    # if this block is true, it implies a priority of 0 (mutually exclusive to previous block)
                    if (counts['total'] < 20):
                        change_whiteboard = self.check_needswork(bug_info[bug_id]['whiteboard'])

                else:
                    change_priority, change_whiteboard = self.check_needswork_owner(bug_info[bug_id])

                # recommend disabling when more than 150 failures tracked over 21 days and
                # takes precedence over any prevous change_whiteboard assignments
                if (bug_id in alt_date_bug_totals and not self.check_whiteboard_status(bug_info[bug_id]['whiteboard'])):
                    priority = 3
                    change_whiteboard = self.update_whiteboard(bug_info[bug_id]['whiteboard'], '[stockwell disable-recommended]')

            comment = template.render(bug_id=bug_id,
                                      total=counts['total'],
                                      test_run_count=test_run_count,
                                      rank=rank,
                                      priority=priority,
                                      failure_rate=round(counts['total']/float(test_run_count), 3),
                                      repositories=counts['per_repository'],
                                      platforms=counts['per_platform'],
                                      startday=startday,
                                      endday=endday.split()[0],
                                      weekly_mode=self.weekly_mode)

            bug_changes = {'bug_id': bug_id,
                           'changes': {
                              'comment': {'body': comment}
                            }
                           }

            if change_whiteboard:
                bug_changes['changes']['whiteboard'] = change_whiteboard

            if change_priority:
                bug_changes['changes']['priority'] = change_priority

            all_bug_changes.append(bug_changes)

        return all_bug_changes
428434
def run(align_bams, items, ref_file, assoc_files, region=None, out_file=None):
    
    paired = vcfutils.get_paired_bams(align_bams, items)
    assert paired and not paired.normal_bam, ("smCounter2 supports tumor-only variant calling: %s" %
                                              (",".join([dd.get_sample_name(d) for d in items])))
    vrs = bedutils.population_variant_regions(items)
    target = shared.subset_variant_regions(vrs, region,
                                            out_file, items=items, do_merge=True)
    out_file = out_file.replace(".vcf.gz", ".vcf")
    out_prefix = utils.splitext_plus(os.path.basename(out_file))[0]
    if not utils.file_exists(out_file) and not utils.file_exists(out_file + ".gz"):
        with file_transaction(paired.tumor_data, out_file) as tx_out_file:
            cmd = ["smCounter2", "--runPath", os.path.dirname(tx_out_file),
                   "--outPrefix", out_prefix,
                   "--bedTarget", target, "--refGenome", ref_file,
                   "--bamFile", paired.tumor_bam, "--bamType", "consensus",
                   "--nCPU", dd.get_num_cores(paired.tumor_data)]
            do.run(cmd, "smcounter2 variant calling")
            for fname in glob.glob(os.path.join(os.path.dirname(tx_out_file), "*.smCounter*")):
                shutil.move(fname, os.path.join(os.path.dirname(out_file), os.path.basename(fname)))
            utils.symlink_plus(os.path.join(os.path.dirname(out_file),
                                            "%s.smCounter.cut.vcf" % out_prefix),
                               out_file)
    return vcfutils.bgzip_and_index(out_file, paired.tumor_data["config"], remove_orig=False,
                                    prep_cmd="sed 's#FORMAT\t%s#FORMAT\t%s#' | %s" %
                                    (out_prefix, dd.get_sample_name(paired.tumor_data),
                                     vcfutils.add_contig_to_header_cl(dd.get_ref_file(paired.tumor_data), out_file)))
1204245
def mutation(self, strength = 0.1):
		'''
		
		'''
		mutStrengthReal = strength
		mutMaxSizeReal = self.gLength/2
		mutSizeReal = int(numpy.random.random_integers(1,mutMaxSizeReal))
		mutationPosReal = int(numpy.random.random_integers(0+mutSizeReal-1,self.y.shape[0]-1-mutSizeReal))
		mutationSignReal = pl.rand()
		mutationReal = pl.rand()

		if mutationSignReal > 0.5:
			for i in range(-mutSizeReal/2, mutSizeReal/2):
				self.y.real[mutationPosReal+i] = self.y.real[mutationPosReal+i] + mutStrengthReal*self.y.real[mutationPosReal+i]*mutationReal
		else:
			for i in range(-mutSizeReal/2, mutSizeReal/2):
				self.y.real[mutationPosReal+i] = self.y.real[mutationPosReal+i] - mutStrengthReal*self.y.real[mutationPosReal+i]*mutationReal

		self.birth()
1373057
private void score0(double[] data, double[] preds, int treeIdx) {
    Key[] keys = _output._treeKeys[treeIdx];
    for( int c=0; c<keys.length; c++ ) {
      if (keys[c] != null) {
        double pred = DKV.get(keys[c]).<CompressedTree>get().score(data,_output._domains);
        assert (!Double.isInfinite(pred));
        preds[keys.length == 1 ? 0 : c + 1] += pred;
      }
    }
  }
775135
function logResults(results) {
		const { violations, testEngine, testEnvironment, testRunner } = results;

		if (violations.length === 0) {
			cliReporter(colors.green('  0 violations found!'));
			return;
		}

		const issueCount = violations.reduce((count, violation) => {
			cliReporter(
				'\n' +
					error('  Violation of %j with %d occurrences!\n') +
					'    %s. Correct invalid elements at:\n' +
					violation.nodes
						.map(node => '     - ' + utils.selectorToString(node.target) + '\n')
						.join('') +
					'    For details, see: %s',
				violation.id,
				violation.nodes.length,
				violation.description,
				link(violation.helpUrl.split('?')[0])
			);
			return count + violation.nodes.length;
		}, 0);

		cliReporter(error('\n%d Accessibility issues detected.'), issueCount);

		if (program.verbose) {
			const metadata = {
				'Test Runner': testRunner,
				'Test Engine': testEngine,
				'Test Environment': testEnvironment
			};
			cliReporter(`\n${JSON.stringify(metadata, null, 2)}`);
		}

		if (program.exit) {
			process.exitCode = 1;
		}
	}
956092
public static int calculateHintTTL(Mutation mutation)
    {
        int ttl = maxHintTTL;
        for (ColumnFamily cf : mutation.getColumnFamilies())
            ttl = Math.min(ttl, cf.metadata().getGcGraceSeconds());
        return ttl;
    }


    public void start()
    {
        MBeanServer mbs = ManagementFactory.getPlatformMBeanServer();
        try
        {
            mbs.registerMBean(this, new ObjectName(MBEAN_NAME));
        }
        catch (Exception e)
        {
            throw new RuntimeException(e);
        }
        logger.debug("Created HHOM instance, registered MBean.");

        Runnable runnable = new Runnable()
        {
            public void run()
            {
                scheduleAllDeliveries();
                metrics.log();
            }
        };
        executor.scheduleWithFixedDelay(runnable, 10, 10, TimeUnit.MINUTES);
    }

    private static void deleteHint(ByteBuffer tokenBytes, CellName columnName, long timestamp)
    {
        Mutation mutation = new Mutation(Keyspace.SYSTEM_KS, tokenBytes);
        mutation.delete(SystemKeyspace.HINTS_CF, columnName, timestamp);
        mutation.applyUnsafe(); // don't bother with commitlog since we're going to flush as soon as we're done with delivery
    }

    public void deleteHintsForEndpoint(final String ipOrHostname)
    {
        try
        {
            InetAddress endpoint = InetAddress.getByName(ipOrHostname);
            deleteHintsForEndpoint(endpoint);
        }
        catch (UnknownHostException e)
        {
            logger.warn("Unable to find {}, not a hostname or ipaddr of a node", ipOrHostname);
            throw new RuntimeException(e);
        }
    }

    public void deleteHintsForEndpoint(final InetAddress endpoint)
    {
        if (!StorageService.instance.getTokenMetadata().isMember(endpoint))
            return;
        UUID hostId = StorageService.instance.getTokenMetadata().getHostId(endpoint);
        ByteBuffer hostIdBytes = ByteBuffer.wrap(UUIDGen.decompose(hostId));
        final Mutation mutation = new Mutation(Keyspace.SYSTEM_KS, hostIdBytes);
        mutation.delete(SystemKeyspace.HINTS_CF, System.currentTimeMillis());

        // execute asynchronously to avoid blocking caller (which may be processing gossip)
        Runnable runnable = new Runnable()
        {
            public void run()
            {
                try
                {
                    logger.info("Deleting any stored hints for {}", endpoint);
                    mutation.apply();
                    hintStore.forceBlockingFlush();
                    compact();
                }
                catch (Exception e)
                {
                    JVMStabilityInspector.inspectThrowable(e);
                    logger.warn("Could not delete hints for {}: {}", endpoint, e);
                }
            }
        };
        executor.submit(runnable);
    }

    //foobar
    public void truncateAllHints() throws ExecutionException, InterruptedException
    {
        Runnable runnable = new Runnable()
        {
            public void run()
            {
                try
                {
                    logger.info("Truncating all stored hints.");
                    Keyspace.open(Keyspace.SYSTEM_KS).getColumnFamilyStore(SystemKeyspace.HINTS_CF).truncateBlocking();
                }
                catch (Exception e)
                {
                    logger.warn("Could not truncate all hints.", e);
                }
            }
        };
        executor.submit(runnable).get();

    }

    @VisibleForTesting
    protected synchronized void compact()
    {
        ArrayList<Descriptor> descriptors = new ArrayList<>();
        for (SSTable sstable : hintStore.getDataTracker().getUncompactingSSTables())
            descriptors.add(sstable.descriptor);

        if (descriptors.isEmpty())
            return;

        try
        {
            CompactionManager.instance.submitUserDefined(hintStore, descriptors, (int) (System.currentTimeMillis() / 1000)).get();
        }
        catch (InterruptedException | ExecutionException e)
        {
            throw new RuntimeException(e);
        }
    }

    private static boolean pagingFinished(ColumnFamily hintColumnFamily, Composite startColumn)
    {
        // done if no hints found or the start column (same as last column processed in previous iteration) is the only one
        return hintColumnFamily == null
               || (!startColumn.isEmpty() && hintColumnFamily.getSortedColumns().size() == 1 && hintColumnFamily.getColumn((CellName)startColumn) != null);
    }
861267
def _run_qc_tools(bam_file, data):
    
    from bcbio.qc import (atropos, contamination, coverage, damage, fastqc, kraken,
                          qsignature, qualimap, samtools, picard, srna, umi, variant,
                          viral, preseq, chipseq)
    tools = {"fastqc": fastqc.run,
             "atropos": atropos.run,
             "small-rna": srna.run,
             "samtools": samtools.run,
             "qualimap": qualimap.run,
             "qualimap_rnaseq": qualimap.run_rnaseq,
             "qsignature": qsignature.run,
             "contamination": contamination.run,
             "coverage": coverage.run,
             "damage": damage.run,
             "variants": variant.run,
             "peddy": peddy.run_qc,
             "kraken": kraken.run,
             "picard": picard.run,
             "umi": umi.run,
             "viral": viral.run,
             "preseq": preseq.run,
             "chipqc": chipseq.run
             }
    qc_dir = utils.safe_makedir(os.path.join(data["dirs"]["work"], "qc", data["description"]))
    metrics = {}
    qc_out = utils.deepish_copy(dd.get_summary_qc(data))
    for program_name in dd.get_algorithm_qc(data):
        if not bam_file and program_name != "kraken":  # kraken doesn't need bam
            continue
        if dd.get_phenotype(data) == "germline" and program_name != "variants":
            continue
        qc_fn = tools[program_name]
        cur_qc_dir = os.path.join(qc_dir, program_name)
        out = qc_fn(bam_file, data, cur_qc_dir)
        qc_files = None
        if out and isinstance(out, dict):
            # Check for metrics output, two cases:
            # 1. output with {"metrics"} and files ("base")
            if "metrics" in out:
                metrics.update(out.pop("metrics"))
            # 2. a dictionary of metrics
            elif "base" not in out:
                metrics.update(out)
            # Check for files only output
            if "base" in out:
                qc_files = out
        elif out and isinstance(out, six.string_types) and os.path.exists(out):
            qc_files = {"base": out, "secondary": []}
        if not qc_files:
            qc_files = _organize_qc_files(program_name, cur_qc_dir)
        if qc_files:
            qc_out[program_name] = qc_files

    metrics["Name"] = dd.get_sample_name(data)
    metrics["Quality format"] = dd.get_quality_format(data).lower()
    return {"qc": qc_out, "metrics": metrics}