Convenience wrapper around mark_best_classification.

    Finds the best match for each TextLogError in errors, handling no match
    meeting the cut off score and then mark_best_classification to save that
    information.
544758
def mark_best_classifications(errors):
    
    for text_log_error in errors:
        best_match = get_best_match(text_log_error)
        if not best_match:
            continue

        mark_best_classification(text_log_error, best_match.classified_failure)
559532
def best_match(given_word)
      words = (@word_list.is_a? Array) ? @word_list : @word_list.keys

      word_bigrams = bigramate(given_word)
      word_hash = words.map do |key|
        [key, bigram_compare(word_bigrams, bigramate(key))]
      end
      word_hash = Hash[word_hash]

      # Weight by word usage, if logical
      word_hash = apply_usage_weights(word_hash) if @word_list.is_a? Hash

      word_hash.max_by { |_key, value| value }.first
    end
440724
def checkError(barcode, whitelist, errors=1):
    '''
    
    '''

    near_matches = []
    comp_regex = regex.compile("(%s){e<=%i}" % (barcode, errors))
    b_length = len(barcode)
    for whitelisted_barcode in whitelist:
        w_length = len(whitelisted_barcode)

        # Don't check against itself
        if barcode == whitelisted_barcode:
            continue

        # If difference in barcode lengths > number of allowed errors, continue
        if (max(b_length, w_length) > (min(b_length, w_length) + errors)):
            continue
        if comp_regex.match(whitelisted_barcode):
            near_matches.append(whitelisted_barcode)

            # Assuming downstream processes are the same for
            # (>1 -> Inf) near_matches this is OK
            if len(near_matches) > 1:
                return near_matches

    return near_matches
1331820
def classify(text)
      result, score = classify_with_score(text)
      result = nil if threshold_enabled? && (score < @threshold || score == Float::INFINITY)
      result
    end
628013
def _check_toplevel_misplaced(item):
    
    problem_keys = [k for k in item.keys() if k in ALGORITHM_KEYS]
    if len(problem_keys) > 0:
        raise ValueError("Unexpected configuration keywords found in top level of %s: %s\n"
                         "This should be placed in the 'algorithm' section."
                         % (item["description"], problem_keys))
    problem_keys = [k for k in item.keys() if k not in TOPLEVEL_KEYS]
    if len(problem_keys) > 0:
        raise ValueError("Unexpected configuration keywords found in top level of %s: %s\n"
                         % (item["description"], problem_keys))
713082
def filter_skiplines(code, errors):
    
    if not errors:
        return errors

    enums = set(er.lnum for er in errors)
    removed = set([
        num for num, l in enumerate(code.split('\n'), 1)
        if num in enums and SKIP_PATTERN(l)
    ])

    if removed:
        errors = [er for er in errors if er.lnum not in removed]

    return errors
992043
def train_classifier(download=True, parameters=None, ngram_range=(1, 1)):
    """"""
    if download:
        download_wiki()

    path = os.path.join(l.TOPDIR, 'train.json')
    training_set = json.load(open(path))
    path = os.path.join(l.TOPDIR, 'wiki.json')
    wiki_set = json.load(open(path))

    target_names = list(set([i['unit'] for i in training_set + wiki_set]))
    train_data, train_target = [], []
    for example in training_set + wiki_set:
        train_data.append(clean_text(example['text']))
        train_target.append(target_names.index(example['unit']))

    tfidf_model = TfidfVectorizer(sublinear_tf=True,
                                  ngram_range=ngram_range,
                                  stop_words='english')

    matrix = tfidf_model.fit_transform(train_data)

    if parameters is None:
        parameters = {'loss': 'log', 'penalty': 'l2', 'n_iter': 50,
                      'alpha': 0.00001, 'fit_intercept': True}

    clf = SGDClassifier(**parameters).fit(matrix, train_target)
    obj = {'tfidf_model': tfidf_model,
           'clf': clf,
           'target_names': target_names}
    path = os.path.join(l.TOPDIR, 'clf.pickle')
    pickle.dump(obj, open(path, 'w'))
1162295
def clear_score(self):
        
        if self._grade_system.is_based_on_grades():
            return  # do nothing, spec does not raise error
        if (self.get_score_metadata().is_read_only() or
                self.get_score_metadata().is_required()):
            raise errors.NoAccess()
        self._my_map['score'] = self._score_default
        self._my_map['gradingAgentId'] = ''
        self._my_map['timeGraded'] = None
100249
def _detect_line_ending(self):
        
        candidate_value = '\n'
        candidate_count = 0
        for line_ending in UniversalCsvReader.line_endings:
            count = self._sample.count(line_ending)
            if count > candidate_count:
                candidate_value = line_ending
                candidate_count = count

        self._formatting_parameters['line_terminator'] = candidate_value
459944
def get_token(self, text, start=0):
        
        best_class = best_match = None

        for token_class, match in self.matching_tokens(text):
            if best_match and best_match.end() >= match.end():
                continue
            best_match = match
            best_class = token_class

        return best_class, best_match
191546
def _intertext_score(full_text):
    ''''''

    sentences = sentence_tokenizer(full_text)
    norm = _normalize(sentences)
    similarity_matrix = pairwise_kernels(norm, metric='cosine')
    scores = _textrank(similarity_matrix)
    scored_sentences = []
    for i, s in enumerate(sentences):
        scored_sentences.append((scores[i],i,s))
    top_scorers = sorted(scored_sentences,
                         key=lambda tup: tup[0], 
                         reverse=True)
    return top_scorers
426632
def ErrorMsg():
    
    import traceback

    limit = None
    _type, value, tb = sys.exc_info()
    _list = traceback.format_tb(tb, limit) + \
        traceback.format_exception_only(_type, value)
    return "Traceback (innermost last):\n" + "%-20s %s" % (
        " ".join(_list[:-1]),
        _list[-1])
426740
def _get_mark_if_any(self):
        """"""
        line = self.next_line()
        if line.startswith(b'mark :'):
            return line[len(b'mark :'):]
        else:
            self.push_line(line)
            return None
1325519
def prune_by_ngram_size(self, minimum=None, maximum=None):
        
        self._logger.info('Pruning results by n-gram size')
        if minimum:
            self._matches = self._matches[
                self._matches[constants.SIZE_FIELDNAME] >= minimum]
        if maximum:
            self._matches = self._matches[
                self._matches[constants.SIZE_FIELDNAME] <= maximum]
744134
def get_error_summary(job):
    
    cache_key = 'error-summary-{}'.format(job.id)
    cached_error_summary = cache.get(cache_key)
    if cached_error_summary is not None:
        return cached_error_summary

    # don't cache or do anything if we have no text log errors to get
    # results for
    errors = TextLogError.objects.filter(step__job=job)
    if not errors:
        return []

    # cache terms generated from error line to save excessive querying
    term_cache = {}

    error_summary = [bug_suggestions_line(err, term_cache) for err in errors]
    cache.set(cache_key, error_summary, BUG_SUGGESTION_CACHE_TIMEOUT)

    return error_summary
632811
def class_wise_error_rate(self, event_label):
        

        deletion_rate = metric.deletion_rate(
            Nref=self.class_wise[event_label]['Nref'],
            Ndeletions=self.class_wise[event_label]['Nfn']
        )

        insertion_rate = metric.insertion_rate(
            Nref=self.class_wise[event_label]['Nref'],
            Ninsertions=self.class_wise[event_label]['Nfp']
        )

        error_rate = metric.error_rate(
            deletion_rate_value=deletion_rate,
            insertion_rate_value=insertion_rate
        )

        return {
            'error_rate': error_rate,
            'deletion_rate': deletion_rate,
            'insertion_rate': insertion_rate
        }
923145
def trace_error(function_index=2):
    
    info = function_info(function_index)
    traces = traceback.format_stack(limit=10)
    for trace in traces:
        file_, line_number, line_text = trace.split(',', 2)
        if file_ == '  File "%s"' % info['file'] and\
                        line_number != 'line %s' % info['line_number']:
            return line_number.split()[-1], line_text.strip()
    return None, None