conducts epsilon lexicase selection for de-aggregated fitness vectors
965704
def epsilon_lexicase(self, F, sizes, num_selections=None, survival = False):
        """"""
        # pdb.set_trace()
        if num_selections is None:
                num_selections = F.shape[0]

        if self.c: # use c library
            # define c types
            locs = np.empty(num_selections,dtype='int32',order='F')
            # self.lib.epsilon_lexicase(F,F.shape[0],F.shape[1],num_selections,locs)
            if self.lex_size:
                ep_lex(F,F.shape[0],F.shape[1],num_selections,locs,self.lex_size,np.array(sizes))
            else:
                ep_lex(F,F.shape[0],F.shape[1],num_selections,locs,self.lex_size,np.array([]))
            return locs
        else: # use python version
            
            locs = []
            individual_locs = np.arange(F.shape[0])
            # calculate epsilon thresholds based on median absolute deviation (MAD)
            mad_for_case = np.array([self.mad(f) for f in F.transpose()])
            for i in np.arange(num_selections):

                can_locs = individual_locs
                cases = list(np.arange(F.shape[1]))
                self.random_state.shuffle(cases)
                # pdb.set_trace()
                while len(cases) > 0 and len(can_locs) > 1:
                    # get best fitness for case among candidates
                    best_val_for_case = np.min(F[can_locs,cases[0]])
                    # filter individuals without an elite fitness on this case
                    can_locs = [l for l in can_locs if F[l,cases[0]] <= best_val_for_case + mad_for_case[cases[0]]]
                    cases.pop(0)

                choice = self.random_state.randint(len(can_locs))
                locs.append(can_locs[choice])
                if survival: # filter out winners from remaining selection pool
                    individual_locs = [i for i in individual_locs if i != can_locs[choice]]

            while len(locs) < num_selections:
                locs.append(individual_locs[0])

            return locs
31071
def from_seqfeature(s, **kwargs):
    
    source = s.qualifiers.get('source', '.')[0]
    score = s.qualifiers.get('score', '.')[0]
    seqid = s.qualifiers.get('seqid', '.')[0]
    frame = s.qualifiers.get('frame', '.')[0]
    strand = _feature_strand[s.strand]

    # BioPython parses 1-based GenBank positions into 0-based for use within
    # Python.  We need to convert back to 1-based GFF format here.
    start = s.location.start.position + 1
    stop = s.location.end.position
    featuretype = s.type
    id = s.id
    attributes = dict(s.qualifiers)
    attributes.pop('source', '.')
    attributes.pop('score', '.')
    attributes.pop('seqid', '.')
    attributes.pop('frame', '.')
    return Feature(seqid, source, featuretype, start, stop, score, strand,
                   frame, attributes, id=id, **kwargs)
345984
def filter_genes_fano_deprecated(X, Ecutoff, Vcutoff):
    
    if issparse(X):
        raise ValueError('Not defined for sparse input. See `filter_genes_dispersion`.')
    mean_filter = np.mean(X, axis=0) > Ecutoff
    var_filter = np.var(X, axis=0) / (np.mean(X, axis=0) + .0001) > Vcutoff
    gene_subset = np.nonzero(np.all([mean_filter, var_filter], axis=0))[0]
    return gene_subset
38298
def _get_fuzzy_tc_matches(text, full_text, options):
    '''
    
    '''
    print("text: {}, full: {}, options: {}".format(text, full_text, options))
    # get the options which match the full text
    matching_options = _get_fuzzy_matches(full_text, options)
    # need to only return the individual words which:
    # - match the 'text'
    # - are not exclusively matched by other input in full_text
    # - when matched, still allows all other input in full_text to be matched
    # get the input tokens
    input_tokens = full_text.split()
    # remove one instance of the text to be matched
    initial_tokens = input_tokens.remove(text)
    # track the final matches:
    final_matches = []
    # find matches per option
    for option in options:
        option_tokens = option.split()
        # get tokens which match the text
        matches = [t for t in option_tokens if text in t]
        # get input tokens which match one of the matches
        input_tokens_which_match = [t for t in input_tokens for m in matches if t in m]
        # if any input token ONLY matches a match, remove that match
        for token in input_tokens_which_match:
            token_matches = [t for t in option_tokens if token in t]
            if len(token_matches) == 1:
                match = token_matches[0]
                if match in matches:
                    matches.remove(match)
        # for the remaining matches, if the input tokens can be fuzzily matched without
        # the match, it's ok to return it.
        for match in matches:
            # copy option tokens
            option_tokens_minus_match = option_tokens[:]
            # remove the match
            option_tokens_minus_match.remove(match)
            option_minus_match = ' '.join(option_tokens_minus_match)
            if _get_fuzzy_matches(' '.join(input_tokens), [option_minus_match]):
                if match not in final_matches:
                    final_matches.append(match)
    return final_matches
590422
def split(*items):
    
    out = []
    for data in [x[0] for x in items]:
        dis_orgs = data["config"]["algorithm"].get("disambiguate")
        if dis_orgs:
            if not data.get("disambiguate", None):
                data["disambiguate"] = {"genome_build": data["genome_build"],
                                        "base": True}
            out.append([data])
            # handle the instance where a single organism is disambiguated
            if isinstance(dis_orgs, six.string_types):
                dis_orgs = [dis_orgs]
            for dis_org in dis_orgs:
                dis_data = copy.deepcopy(data)
                dis_data["disambiguate"] = {"genome_build": dis_org}
                dis_data["genome_build"] = dis_org
                dis_data["config"]["algorithm"]["effects"] = False
                dis_data = run_info.add_reference_resources(dis_data)
                out.append([dis_data])
        else:
            out.append([data])
    return out
651773
def genOutputs(self, code, match):
        

        out = sorted((k, match.output(m)) for (k, m) in
                     self.collectTriggers(match.match, code).items())
        out = list(map(lambda a: a[1], out))
        return out
1221423
function getRelevantIndexes(text, inlineRanges, entityRanges = [], decoratorRanges = []) {
  let relevantIndexes = [];
  // set indexes to corresponding keys to ensure uniquenes
  relevantIndexes = addIndexes(relevantIndexes, inlineRanges);
  relevantIndexes = addIndexes(relevantIndexes, entityRanges);
  relevantIndexes = addIndexes(relevantIndexes, decoratorRanges);
  // add text start and end to relevant indexes
  relevantIndexes.push(0);
  relevantIndexes.push(text.length);
  const uniqueRelevantIndexes = relevantIndexes.filter(
    (value, index, self) => self.indexOf(value) === index
  );
  // and sort it
  return uniqueRelevantIndexes.sort((aa, bb) => (aa - bb));
}
111874
def evaluate(self, genomes, config):
        """"""
        if not self.working:
            self.start()
        p = 0
        for genome_id, genome in genomes:
            p += 1
            self.inqueue.put((genome_id, genome, config))

        # assign the fitness back to each genome
        while p > 0:
            p -= 1
            ignored_genome_id, genome, fitness = self.outqueue.get()
            genome.fitness = fitness
1018496
def preproc_data(data, gene_subset=False, **kwargs):
    
    import uncurl
    from uncurl.preprocessing import log1p, cell_normalize
    from sklearn.decomposition import TruncatedSVD
    data_subset = data
    if gene_subset:
        gene_subset = uncurl.max_variance_genes(data)
        data_subset = data[gene_subset, :]
    tsvd = TruncatedSVD(min(8, data_subset.shape[0] - 1))
    data_tsvd = tsvd.fit_transform(log1p(cell_normalize(data_subset)).T)
    return data_tsvd
792320
def get_meta_regex(schema='mona'):
    
    # NOTE: will just ignore cases, to avoid repetition here
    meta_parse = collections.OrderedDict()

    if schema == 'mona':
        meta_parse['collision_energy'] = ['^collision energy(?:=|:)(.*)$']
        meta_parse['ms_level'] = ['^ms.*level(?:=|:)\D*(\d*)$', '^ms type(?:=|:)\D*(\d*)$',
                              '^Spectrum_type(?:=|:)\D*(\d*)$']
        meta_parse['accession'] = ['^accession(?:=|:)(.*)$', '^DB#(?:=|:)(.*)$']
        meta_parse['resolution'] = ['^resolution(?:=|:)(.*)$']
        meta_parse['polarity'] = ['^ion.*mode(?:=|:)(.*)$', '^ionization.*mode(?:=|:)(.*)$', '^polarity(?:=|:)(.*)$']
        meta_parse['fragmentation_type'] = ['^fragmentation.*mode(?:=|:)(.*)$', '^fragmentation.*type(?:=|:)(.*)$']
        meta_parse['precursor_mz'] = ['^precursor m/z(?:=|:)\s*(\d*[.,]?\d*)$', '^precursor.*mz(?:=|:)\s*(\d*[.,]?\d*)$']
        meta_parse['precursor_type'] = ['^precursor.*type(?:=|:)(.*)$', '^adduct(?:=|:)(.*)$']
        meta_parse['instrument_type'] = ['^instrument.*type(?:=|:)(.*)$']
        meta_parse['instrument'] = ['^instrument(?:=|:)(.*)$']
        meta_parse['copyright'] = ['^copyright(?:=|:)(.*)$']
        # meta_parse['column'] = ['^column(?:=|:)(.*)$']
        meta_parse['mass_accuracy'] = ['^mass.*accuracy(?:=|:)\s*(\d*[.,]?\d*)$']
        meta_parse['mass_error'] = ['^mass.*error(?:=|:)\s*(\d*[.,]?\d*)$']
        meta_parse['origin'] = ['^origin(?:=|:)(.*)$']
        meta_parse['name'] = ['^Name(?:=|:)(.*)$']
        meta_parse['splash'] = ['^splash:(.*)$']
        meta_parse['retention_time'] = ['^retention.*time(?:=|:)\s*(\d*[.,]?\d*)$']
        meta_parse['retention_index'] = ['^retention.*index(?:=|:)\s*(\d*[.,]?\d*)$']

    elif schema == 'massbank':
        meta_parse['collision_energy'] = ['^AC\$MASS_SPECTROMETRY:\s+COLLISION_ENERGY\s+(.*)$']
        meta_parse['ms_level'] = ['^AC\$MASS_SPECTROMETRY:\s+MS_TYPE\s+\D*(\d*)$']
        meta_parse['accession'] = ['^ACCESSION:(.*)$']
        meta_parse['resolution'] = ['^AC\$MASS_SPECTROMETRY:\s+RESOLUTION\s+(.*)$']
        meta_parse['polarity'] = ['^AC\$MASS_SPECTROMETRY:\s+ION_MODE\s+(.*)$']
        meta_parse['fragmentation_type'] = ['^AC\$MASS_SPECTROMETRY:\s+FRAGMENTATION_MODE\s+(.*)$']
        meta_parse['precursor_mz'] = ['^MS\$FOCUSED_ION:\s+PRECURSOR_M/Z\s+(\d*[.,]?\d*)$']
        meta_parse['precursor_type'] = ['^MS\$FOCUSED_ION:\s+PRECURSOR_TYPE\s+(.*)$']
        meta_parse['instrument_type'] = ['^AC\$INSTRUMENT_TYPE:\s+(.*)$']
        meta_parse['instrument'] = ['^AC\$INSTRUMENT:\s+(.*)$']
        meta_parse['copyright'] = ['^COPYRIGHT:\s+(.*)']
        # meta_parse['column'] = ['^column(?:=|:)(.*)$']
        meta_parse['mass_accuracy'] = ['^AC\$MASS_SPECTROMETRY:\s+ACCURACY\s+(.*)$']  # need to check
        meta_parse['mass_error'] = ['^AC\$MASS_SPECTROMETRY:\s+ERROR\s+(.*)$']  # need to check
        meta_parse['splash'] = ['^PK\$SPLASH:\s+(.*)$']
        meta_parse['origin'] = ['^origin(?:=|:)(.*)$']
        meta_parse['name'] = ['^RECORD_TITLE:\s+(.*)$']
        meta_parse['retention_time'] = ['^AC\$CHROMATOGRAPHY:\s+RETENTION.*TIME\s+(\d*[.,]?\d*)$']
        meta_parse['retention_index'] = ['^AC\$CHROMATOGRAPHY:\s+RETENTION.*INDEX\s+(\d*[.,]?\d*)$']




    return meta_parse
39261
protected static void calculateSelectivityCoeffs(List<DoubleObjPair<DAFile>> daFiles, NumberVector query, double epsilon) {
    final int dimensions = query.getDimensionality();
    double[] lowerVals = new double[dimensions];
    double[] upperVals = new double[dimensions];

    VectorApproximation queryApprox = calculatePartialApproximation(null, query, daFiles);

    for(int i = 0; i < dimensions; i++) {
      final double val = query.doubleValue(i);
      lowerVals[i] = val - epsilon;
      upperVals[i] = val + epsilon;
    }

    DoubleVector lowerEpsilon = DoubleVector.wrap(lowerVals);
    VectorApproximation lowerEpsilonPartitions = calculatePartialApproximation(null, lowerEpsilon, daFiles);

    DoubleVector upperEpsilon = DoubleVector.wrap(upperVals);
    VectorApproximation upperEpsilonPartitions = calculatePartialApproximation(null, upperEpsilon, daFiles);

    for(int i = 0; i < daFiles.size(); i++) {
      int coeff = (queryApprox.getApproximation(i) - lowerEpsilonPartitions.getApproximation(i)) + (upperEpsilonPartitions.getApproximation(i) - queryApprox.getApproximation(i)) + 1;
      daFiles.get(i).first = coeff;
    }
  }
570716
def rankingEval(self):
		'''
		
		'''
		fitnessAll = numpy.zeros(self.length)
		fitnessNorm = numpy.zeros(self.length)

		for i in range(self.length):
			self.Ind[i].fitnessEval()
			fitnessAll[i] = self.Ind[i].fitness

		maxFitness = fitnessAll.max()
		for i in range(self.length):
			fitnessNorm[i] = (maxFitness - fitnessAll[i]) / maxFitness
		fitnessSorted = fitnessNorm.argsort()

		# Compute the selection probabilities of each individual
		probability = numpy.zeros(self.length)
		S = 2.0
		for i in range(self.length):
			probability[fitnessSorted[i]] = ((2-S)/self.length) + (2*i*(S-1))/(self.length*(self.length-1))
		self.rankingComputed = 1
		self.fitness = 	fitnessAll

		return [fitnessAll, fitnessSorted[::-1], probability]
347475
def survival(self,parents,offspring,elite=None,elite_index=None,X=None,X_O=None,F=None,F_O=None):
        """"""
        if self.sel == 'tournament':
            survivors, survivor_index = self.tournament(parents + offspring, self.tourn_size, num_selections = len(parents))
        elif self.sel == 'lexicase':
            survivor_index = self.lexicase(np.vstack((F,F_O)), num_selections = len(parents), survival = True)
            survivors = [(parents+ offspring)[s] for s in survivor_index]
        elif self.sel == 'epsilon_lexicase':
            # survivors, survivor_index = self.epsilon_lexicase(parents + offspring, num_selections = len(parents), survival = True)
            if self.lex_size:
                sizes = [len(i.stack) for i in (parents + offspring)]
                survivor_index = self.epsilon_lexicase(np.vstack((F,F_O)), sizes, num_selections = F.shape[0], survival = True)
                survivors = [(parents+ offspring)[s] for s in survivor_index]
            else:
                survivor_index = self.epsilon_lexicase(np.vstack((F,F_O)), [], num_selections = F.shape[0], survival = True)
                survivors = [(parents+ offspring)[s] for s in survivor_index]
        elif self.sel == 'deterministic_crowding':
            survivors, survivor_index = self.deterministic_crowding(parents,offspring,X,X_O)
        elif self.sel == 'random':
            # pdb.set_trace()
            survivor_index = self.random_state.permutation(np.arange(2*len(parents)))[:len(parents)]
            survivors = [(parents + offspring)[s] for s in survivor_index]
        # elitism
        if self.elitism:
            if min([x.fitness for x in survivors]) > elite.fitness:
                # if the elite individual did not survive and elitism is on, replace worst individual with elite
                rep_index = np.argmax([x.fitness for x in survivors])
                survivors[rep_index] = elite
                survivor_index[rep_index] = elite_index
        # return survivors

        return survivors,survivor_index
849672
def calc_fitness(self,X,labels,fit_choice,sel):
        

        if 'lexicase' in sel:
            # return list(map(lambda yhat: self.f_vec[fit_choice](labels,yhat),X))
            return np.asarray(
                              [self.proper(self.f_vec[fit_choice](labels,
                                                        yhat)) for yhat in X],
                                                        order='F')
            # return list(Parallel(n_jobs=-1)(delayed(self.f_vec[fit_choice])(labels,yhat) for yhat in X))
        else:
            # return list(map(lambda yhat: self.f[fit_choice](labels,yhat),X))
            return np.asarray([self.f[fit_choice](labels,yhat) for yhat in X],
                            order='F').reshape(-1)
545440
def _fitnesses_to_probabilities(fitnesses):
    """"""
    # Do not allow negative fitness values
    min_fitness = min(fitnesses)
    if min_fitness < 0.0:
        # Make smallest fitness value 0
        fitnesses = map(lambda f: f - min_fitness, fitnesses)

    fitness_sum = sum(fitnesses)

    # Generate probabilities
    # Creates a list of increasing values.
    # The greater the gap between two values, the greater the probability.
    # Ex. [0.1, 0.23, 0.56, 1.0]
    prob_sum = 0.0
    probabilities = []
    for fitness in fitnesses:
        if fitness < 0:
            raise ValueError(
                "Fitness cannot be negative, fitness = {}.".format(fitness))
        prob_sum += (fitness / fitness_sum)
        probabilities.append(prob_sum)
    probabilities[-1] += 0.0001  # to compensate for rounding errors

    return probabilities
1300435
def _get_features(self, i, context, prev, prev2):
        """"""
        w = self.lexicon[context[i]]
        features = [
            'bias',
            'w:shape=%s' % w.shape,
            'w:lower=%s' % w.lower,
            'p1:tag=%s' % prev,
            'p2:tag=%s' % prev2,
            'p1:tag+w:lower=%s+%s' % (prev, w.lower),
            'p1:tag+p2:tag=%s+%s' % (prev, prev2),
        ]
        if w.like_number:
            features.append('w:like_number')
        elif w.is_punct:
            features.append('w:is_punct')
        elif w.like_url:
            features.append('w:like_url')
        else:
            features.extend([
                'w:suffix2=%s' % w.lower[-2:],
                'w:suffix3=%s' % w.lower[-3:],
                'w:suffix4=%s' % w.lower[-4:],
                'w:suffix5=%s' % w.lower[-5:],
                'w:prefix1=%s' % w.lower[:1],
                'w:prefix2=%s' % w.lower[:2],
                'w:prefix3=%s' % w.lower[:3],
            ])
            if w.is_alpha:
                features.append('w:is_alpha')
            elif w.is_hyphenated:
                features.append('w:is_hyphenated')
            if w.is_upper:
                features.append('w:is_upper')
            elif w.is_lower:
                features.append('w:is_lower')
            elif w.is_title:
                features.append('w:is_title')
        if self.clusters and w.cluster:
            features.extend([
                'w:cluster4=%s' % w.cluster[:4],
                'w:cluster6=%s' % w.cluster[:6],
                'w:cluster10=%s' % w.cluster[:10],
                'w:cluster20=%s' % w.cluster[:20],
            ])
        # Add features for previous tokens if present
        if i > 0:
            p1 = self.lexicon[context[i-1]]
            features.extend([
                'p1:lower=%s' % p1.lower,
                'p1:shape=%s' % p1.shape,
            ])
            if not (p1.like_number or p1.is_punct or p1.like_url):
                features.append('p1:suffix3=%s' % p1.lower[-3:])
            if self.clusters and p1.cluster:
                features.extend([
                    'p1:cluster4=%s' % p1.cluster[:4],
                    'p1:cluster6=%s' % p1.cluster[:6],
                    'p1:cluster10=%s' % p1.cluster[:10],
                    'p1:cluster20=%s' % p1.cluster[:20],
                ])
            if i > 1:
                p2 = self.lexicon[context[i-2]]
                features.extend([
                    'p2:lower=%s' % p2.lower,
                    'p2:shape=%s' % p2.shape,
                ])
                if self.clusters and p2.cluster:
                    features.extend([
                        'p2:cluster4=%s' % p2.cluster[:4],
                        'p2:cluster6=%s' % p2.cluster[:6],
                        'p2:cluster10=%s' % p2.cluster[:10],
                        'p2:cluster20=%s' % p2.cluster[:20],
                    ])
        # Add features for next tokens if present
        end = len(context) - 1
        if i < end:
            n1 = self.lexicon[context[i+1]]
            features.extend([
                'n1:lower=%s' % n1.lower,
                'n1:shape=%s' % n1.shape
            ])
            if not (n1.like_number or n1.is_punct or n1.like_url):
                features.append('n1:suffix3=%s' % n1.lower[-3:])
            if self.clusters and n1.cluster:
                features.extend([
                    'n1:cluster4=%s' % n1.cluster[:4],
                    'n1:cluster6=%s' % n1.cluster[:6],
                    'n1:cluster10=%s' % n1.cluster[:10],
                    'n1:cluster20=%s' % n1.cluster[:20],
                ])
            if i < end - 1:
                n2 = self.lexicon[context[i+2]]
                features.extend([
                    'n2:lower=%s' % n2.lower,
                    'n2:shape=%s' % n2.shape
                ])
                if self.clusters and n2.cluster:
                    features.extend([
                        'n2:cluster4=%s' % n2.cluster[:4],
                        'n2:cluster6=%s' % n2.cluster[:6],
                        'n2:cluster10=%s' % n2.cluster[:10],
                        'n2:cluster20=%s' % n2.cluster[:20],
                    ])
        # Add position features
        if i == 0:
            features.append('-firsttoken-')
        elif i == 1:
            features.append('-secondtoken-')
        elif i == end - 1:
            features.append('-secondlasttoken-')
        elif i == end:
            features.append('-lasttoken-')
        return features
389298
def eval_fitness(self, chromosome):
        
        coefficients = self.translator.translate_chromosome(chromosome)
        solution_y = self.compute_y(coefficients, self.num_x)
        fitness = -1 * self.compute_err(solution_y, coefficients)

        return fitness