Combine results from different batches of filtering
193010
def combine_results(self, results):
        """"""
        result = {}
        for key in results[0]:
            result[key] = numpy.concatenate([r[key] for r in results])
        return result
18883
public void filter(CaptureSearchResults results) {
		Iterator<CaptureSearchResult> itr = results.iterator();
		while(itr.hasNext()) {
			CaptureSearchResult result = itr.next();
			String captureDate = result.getCaptureTimestamp();
			if((captureDate.compareTo(startDateStr) >= 0) 
					&& (captureDate.compareTo(endDateStr) < 0)) {
				matches.add(result);
			}		
		}
	}
992162
def samples_to_batches(samples: Iterable, batch_size: int):
        """"""
        it = iter(samples)
        while True:
            with suppress(StopIteration):
                batch_in, batch_out = [], []
                for i in range(batch_size):
                    sample_in, sample_out = next(it)
                    batch_in.append(sample_in)
                    batch_out.append(sample_out)
            if not batch_in:
                raise StopIteration
            yield np.array(batch_in), np.array(batch_out)
905503
def iterEdges(self, filterSelfLoops=False):
        
        for p in self.iterPorts():
            yield from p.iterEdges(filterSelfLoops=filterSelfLoops)
253461
def _join_results(self, results, coarse, join):
    '''
    
    '''
    rval = OrderedDict()
    i_keys = set()
    for res in results:
      i_keys.update( res.keys() )
    for i_key in sorted(i_keys):
      if coarse:
        rval[i_key] = join( [res.get(i_key) for res in results] )
      else:
        rval[i_key] = OrderedDict()
        r_keys = set()
        for res in results:
          r_keys.update( res.get(i_key,{}).keys() )
        for r_key in sorted(r_keys):
          rval[i_key][r_key] = join( [res.get(i_key,{}).get(r_key) for res in results] )
    return rval
804205
def _group_by_sample_and_batch(samples):
    
    out = collections.defaultdict(list)
    for data in samples:
        out[(dd.get_sample_name(data), dd.get_align_bam(data), tuple(_get_batches(data)))].append(data)
    return [xs[0] for xs in out.values()]
413213
def batch(byte_array, funcs):
    
    result = []
    length = bytes_to_int(byte_array[0:4])
    item_size = bytes_to_int(byte_array[4:8])
    for i in range(0, length):
        chunk = byte_array[8+i*item_size:8+(i+1)*item_size]
        for f in funcs:
            f(chunk)
    return result
462006
function filterResults(results, maximum) {
  var filteredResults = []
  var indices = []
  var matrix = {}
  var column
  var key
  var score
  var interpolated
  var index
  var otherIndex
  var maxScore

  for (key in results) {
    score = results[key].score

    if (!matrix[score]) {
      matrix[score] = []
      indices.push(score)
    }

    matrix[score].push(results[key])
  }

  indices.sort(reverse)

  maxScore = indices[0]

  index = -1

  while (indices[++index]) {
    score = indices[index]
    column = matrix[score]

    interpolated = score / maxScore
    otherIndex = -1

    while (column[++otherIndex]) {
      column[otherIndex].score = interpolated
    }

    filteredResults = filteredResults.concat(column)

    if (filteredResults.length >= maximum) {
      break
    }
  }

  return filteredResults
}
1290866
def combine_runs(self):
        

        # Make sure we have a run to add.
        if len(self.new_id) == 0:
            raise ValueError("No new samples are currently saved.")

        # Grab results from saved run.
        saved_id = np.array(self.saved_id)
        saved_u = np.array(self.saved_u)
        saved_v = np.array(self.saved_v)
        saved_logl = np.array(self.saved_logl)
        saved_nc = np.array(self.saved_nc)
        saved_boundidx = np.array(self.saved_boundidx)
        saved_it = np.array(self.saved_it)
        saved_n = np.array(self.saved_n)
        saved_bounditer = np.array(self.saved_bounditer)
        saved_scale = np.array(self.saved_scale)
        saved_batch = np.array(self.saved_batch)
        nsaved = len(saved_n)

        # Grab results from new run.
        new_id = np.array(self.new_id) + max(saved_id) + 1
        new_u = np.array(self.new_u)
        new_v = np.array(self.new_v)
        new_logl = np.array(self.new_logl)
        new_nc = np.array(self.new_nc)
        new_boundidx = np.array(self.new_boundidx)
        new_it = np.array(self.new_it)
        new_n = np.array(self.new_n)
        new_bounditer = np.array(self.new_bounditer)
        new_scale = np.array(self.new_scale)
        nnew = len(new_n)
        llmin, llmax = self.new_logl_min, self.new_logl_max

        # Reset saved results.
        self.saved_id = []
        self.saved_u = []
        self.saved_v = []
        self.saved_logl = []
        self.saved_logvol = []
        self.saved_logwt = []
        self.saved_logz = []
        self.saved_logzvar = []
        self.saved_h = []
        self.saved_nc = []
        self.saved_boundidx = []
        self.saved_it = []
        self.saved_n = []
        self.saved_bounditer = []
        self.saved_scale = []
        self.saved_batch = []

        # Start our counters at the beginning of each set of dead points.
        idx_saved, idx_new = 0, 0  # start of our dead points
        logl_s, logl_n = saved_logl[idx_saved], new_logl[idx_new]
        nlive_s, nlive_n = saved_n[idx_saved], new_n[idx_new]

        # Iteratively walk through both set of samples to simulate
        # a combined run.
        ntot = nsaved + nnew
        logvol = 0.
        for i in range(ntot):
            if logl_s > self.new_logl_min:
                # If our saved samples are past the lower log-likelihood
                # bound, both runs are now "active" and should be used.
                nlive = nlive_s + nlive_n
            else:
                # If instead our collection of dead points are below
                # the bound, just use our collection of saved samples.
                nlive = nlive_s
            # Increment our position along depending on
            # which dead point (saved or new) is worse.
            if logl_s <= logl_n:
                self.saved_id.append(saved_id[idx_saved])
                self.saved_u.append(saved_u[idx_saved])
                self.saved_v.append(saved_v[idx_saved])
                self.saved_logl.append(saved_logl[idx_saved])
                self.saved_nc.append(saved_nc[idx_saved])
                self.saved_boundidx.append(saved_boundidx[idx_saved])
                self.saved_it.append(saved_it[idx_saved])
                self.saved_bounditer.append(saved_bounditer[idx_saved])
                self.saved_scale.append(saved_scale[idx_saved])
                self.saved_batch.append(saved_batch[idx_saved])
                idx_saved += 1
            else:
                self.saved_id.append(new_id[idx_new])
                self.saved_u.append(new_u[idx_new])
                self.saved_v.append(new_v[idx_new])
                self.saved_logl.append(new_logl[idx_new])
                self.saved_nc.append(new_nc[idx_new])
                self.saved_boundidx.append(new_boundidx[idx_new])
                self.saved_it.append(new_it[idx_new])
                self.saved_bounditer.append(new_bounditer[idx_new])
                self.saved_scale.append(new_scale[idx_new])
                self.saved_batch.append(self.batch + 1)
                idx_new += 1

            # Save the number of live points and expected ln(volume).
            logvol -= math.log((nlive + 1.) / nlive)
            self.saved_n.append(nlive)
            self.saved_logvol.append(logvol)

            # Attempt to step along our samples. If we're out of samples,
            # set values to defaults.
            try:
                logl_s = saved_logl[idx_saved]
                nlive_s = saved_n[idx_saved]
            except:
                logl_s = np.inf
                nlive_s = 0
            try:
                logl_n = new_logl[idx_new]
                nlive_n = new_n[idx_new]
            except:
                logl_n = np.inf
                nlive_n = 0

        # Compute quantities of interest.
        h = 0.
        logz = -1.e300
        loglstar = -1.e300
        logzvar = 0.
        logvols_pad = np.concatenate(([0.], self.saved_logvol))
        logdvols = misc.logsumexp(a=np.c_[logvols_pad[:-1], logvols_pad[1:]],
                                  axis=1, b=np.c_[np.ones(ntot),
                                                  -np.ones(ntot)])
        logdvols += math.log(0.5)
        dlvs = logvols_pad[:-1] - logvols_pad[1:]
        for i in range(ntot):
            loglstar_new = self.saved_logl[i]
            logdvol, dlv = logdvols[i], dlvs[i]
            logwt = np.logaddexp(loglstar_new, loglstar) + logdvol
            logz_new = np.logaddexp(logz, logwt)
            lzterm = (math.exp(loglstar - logz_new) * loglstar +
                      math.exp(loglstar_new - logz_new) * loglstar_new)
            h_new = (math.exp(logdvol) * lzterm +
                     math.exp(logz - logz_new) * (h + logz) -
                     logz_new)
            dh = h_new - h
            h = h_new
            logz = logz_new
            logzvar += dh * dlv
            loglstar = loglstar_new
            self.saved_logwt.append(logwt)
            self.saved_logz.append(logz)
            self.saved_logzvar.append(logzvar)
            self.saved_h.append(h)

        # Reset results.
        self.new_id = []
        self.new_u = []
        self.new_v = []
        self.new_logl = []
        self.new_nc = []
        self.new_it = []
        self.new_n = []
        self.new_boundidx = []
        self.new_bounditer = []
        self.new_scale = []
        self.new_logl_min, self.new_logl_max = -np.inf, np.inf

        # Increment batch counter.
        self.batch += 1

        # Saved batch quantities.
        self.saved_batch_nlive.append(max(new_n))
        self.saved_batch_bounds.append((llmin, llmax))
448704
def _combine(self, other, conn='and'):
        
        f = F()

        self_filters = copy.deepcopy(self.filters)
        other_filters = copy.deepcopy(other.filters)

        if not self.filters:
            f.filters = other_filters
        elif not other.filters:
            f.filters = self_filters
        elif conn in self.filters[0]:
            f.filters = self_filters
            f.filters[0][conn].extend(other_filters)
        elif conn in other.filters[0]:
            f.filters = other_filters
            f.filters[0][conn].extend(self_filters)
        else:
            f.filters = [{conn: self_filters + other_filters}]

        return f
358048
def limit_results(partial_results, limiter_method, cumulator_method)
      return [] if partial_results.empty? # guard clause
      results_allowed = results_allowed_from(partial_results, limiter_method)
      cumulate_total(results_allowed.length, cumulator_method)
      results_allowed
    end
1285332
def _merge_filters(self) -> None:
        """"""
        for opts in (["-filter:a", "-af"], ["-filter:v", "-vf"]):
            filter_list = []
            new_argv = []
            cmd_iter = iter(self._argv)
            for element in cmd_iter:
                if element in opts:
                    filter_list.insert(0, next(cmd_iter))
                else:
                    new_argv.append(element)

            # update argv if changes
            if filter_list:
                new_argv.extend([opts[0], ",".join(filter_list)])
                self._argv = new_argv.copy()
409014
function combineStatsAndIndexes(done, results) {
  var indexes = results.getIndexes;
  var stats = results.getIndexStats;
  var sizes = results.getIndexSizes;
  _.each(indexes, function(idx, i) {
    _.assign(indexes[i], stats[idx.name]);
    _.assign(indexes[i], sizes[idx.name]);
  });
  done(null, indexes);
}
924975
def batch_iter(data, batch_size, num_epochs):
    """"""
    data = np.array(data)
    data_size = len(data)
    num_batches_per_epoch = int(len(data)/batch_size) + 1
    for epoch in range(num_epochs):
        # Shuffle the data at each epoch
        shuffle_indices = np.random.permutation(np.arange(data_size))
        shuffled_data = data[shuffle_indices]
        for batch_num in range(num_batches_per_epoch):
            start_index = batch_num * batch_size
            end_index = min((batch_num + 1) * batch_size, data_size)
            yield shuffled_data[start_index:end_index]
1330570
def _combine_sample_regions_batch(batch, items):
    
    config = items[0]["config"]
    work_dir = utils.safe_makedir(os.path.join(items[0]["dirs"]["work"], "regions"))
    analysis_file = os.path.join(work_dir, "%s-analysis_blocks.bed" % batch)
    no_analysis_file = os.path.join(work_dir, "%s-noanalysis_blocks.bed" % batch)
    if not utils.file_exists(analysis_file) or _needs_region_update(analysis_file, items):
        # Combine all nblocks into a final set of intersecting regions
        # without callable bases. HT @brentp for intersection approach
        # https://groups.google.com/forum/?fromgroups#!topic/bedtools-discuss/qA9wK4zN8do
        bed_regions = [pybedtools.BedTool(x["regions"]["nblock"])
                       for x in items if "regions" in x and x["regions"]["nblock"]]
        if len(bed_regions) == 0:
            analysis_file, no_analysis_file = None, None
        else:
            with file_transaction(items[0], analysis_file, no_analysis_file) as (tx_afile, tx_noafile):
                def intersect_two(a, b):
                    return a.intersect(b, nonamecheck=True).saveas()
                nblock_regions = reduce(intersect_two, bed_regions).saveas(
                    "%s-nblock%s" % utils.splitext_plus(tx_afile))
                ref_file = tz.get_in(["reference", "fasta", "base"], items[0])
                ref_regions = get_ref_bedtool(ref_file, config)
                min_n_size = int(config["algorithm"].get("nomap_split_size", 250))
                block_filter = NBlockRegionPicker(ref_regions, config, min_n_size)
                final_nblock_regions = nblock_regions.filter(
                    block_filter.include_block).saveas().each(block_filter.expand_block).saveas(
                        "%s-nblockfinal%s" % utils.splitext_plus(tx_afile))
                final_regions = ref_regions.subtract(final_nblock_regions, nonamecheck=True).\
                                saveas().merge(d=min_n_size)
                _write_bed_regions(items[0], final_regions, tx_afile, tx_noafile)
    if analysis_file and utils.file_exists(analysis_file):
        return analysis_file, no_analysis_file
    else:
        return None, None
980200
def combine_calls(*args):
    
    if len(args) == 3:
        is_cwl = False
        batch_id, samples, data = args
        caller_names, vrn_files = _organize_variants(samples, batch_id)
    else:
        is_cwl = True
        samples = [utils.to_single_data(x) for x in args]
        samples = [cwlutils.unpack_tarballs(x, x) for x in samples]
        data = samples[0]
        batch_id = data["batch_id"]
        caller_names = data["variants"]["variantcallers"]
        vrn_files = data["variants"]["calls"]
    logger.info("Ensemble consensus calls for {0}: {1}".format(
        batch_id, ",".join(caller_names)))
    edata = copy.deepcopy(data)
    base_dir = utils.safe_makedir(os.path.join(edata["dirs"]["work"], "ensemble", batch_id))
    if any([vcfutils.vcf_has_variants(f) for f in vrn_files]):
        # Decompose multiallelic variants and normalize
        passonly = not tz.get_in(["config", "algorithm", "ensemble", "use_filtered"], edata, False)
        vrn_files = [normalize.normalize(f, data, passonly=passonly, rerun_effects=False, remove_oldeffects=True,
                                         nonrefonly=True,
                                         work_dir=utils.safe_makedir(os.path.join(base_dir, c)))
                     for c, f in zip(caller_names, vrn_files)]
        if "classifiers" not in (dd.get_ensemble(edata) or {}):
            callinfo = _run_ensemble_intersection(batch_id, vrn_files, caller_names, base_dir, edata)
        else:
            config_file = _write_config_file(batch_id, caller_names, base_dir, edata)
            callinfo = _run_ensemble(batch_id, vrn_files, config_file, base_dir,
                                     dd.get_ref_file(edata), edata)
            callinfo["vrn_file"] = vcfutils.bgzip_and_index(callinfo["vrn_file"], data["config"])
        # After decomposing multiallelic variants and normalizing, re-evaluate effects
        ann_ma_file, _ = effects.add_to_vcf(callinfo["vrn_file"], data)
        if ann_ma_file:
            callinfo["vrn_file"] = ann_ma_file

        edata["config"]["algorithm"]["variantcaller"] = "ensemble"
        edata["vrn_file"] = callinfo["vrn_file"]
        edata["ensemble_bed"] = callinfo["bed_file"]
        callinfo["validate"] = validate.compare_to_rm(edata)[0][0].get("validate")
    else:
        out_vcf_file = os.path.join(base_dir, "{0}-ensemble.vcf".format(batch_id))
        vcfutils.write_empty_vcf(out_vcf_file, samples=[dd.get_sample_name(d) for d in samples])
        callinfo = {"variantcaller": "ensemble",
                    "vrn_file": vcfutils.bgzip_and_index(out_vcf_file, data["config"]),
                    "bed_file": None}
    if is_cwl:
        callinfo["batch_samples"] = data["batch_samples"]
        callinfo["batch_id"] = batch_id
        return [{"ensemble": callinfo}]
    else:
        return [[batch_id, callinfo]]
1178125
public void combine(Vector new_filters, int op) throws DBException
    {
        // If this is not a logical operator, throw an exception
        if ((op & LOGICAL_OPER_MASK) == 0)
        {
            throw new DBException();
            // Create a new vector consisting of just the filters
            // from the SearchFilter classes in new_filters
        }
        Vector filters = new Vector();
        // Now add in all the nodes of the new filters
        for (Enumeration e = new_filters.elements(); e.hasMoreElements();)
        {
            // Get the search filter from the vector
            SearchFilter f = (SearchFilter) e.nextElement();
            filters.addElement(f.getFilter());
        }
        // Create a node for this list and return it
        m_filter = new SearchBaseNode(op, m_filter, filters);
    }