returns tuple of scored sentences
       in order of appearance
       Note: Doing an A/B test to
       compare results, reverting to 
       original algorithm.
191546
def _intertext_score(full_text):
    ''''''

    sentences = sentence_tokenizer(full_text)
    norm = _normalize(sentences)
    similarity_matrix = pairwise_kernels(norm, metric='cosine')
    scores = _textrank(similarity_matrix)
    scored_sentences = []
    for i, s in enumerate(sentences):
        scored_sentences.append((scores[i],i,s))
    top_scorers = sorted(scored_sentences,
                         key=lambda tup: tup[0], 
                         reverse=True)
    return top_scorers
394494
def sentences(self):
        """"""
        prevp = 0
        prevs = 0
        sentence = [];
        sentence_id = ""
        for word, id, pos, lemma in iter(self):
            try:
                doc_id, ptype, p, s, w = re.findall('([\w\d-]+)\.(p|head)\.(\d+)\.s\.(\d+)\.w\.(\d+)',id)[0]
                if ((p != prevp) or (s != prevs)) and sentence:
                    yield sentence_id, sentence
                    sentence = []
                    sentence_id = doc_id + '.' + ptype + '.' + str(p) + '.s.' + str(s)
                prevp = p
            except IndexError:
                doc_id, s, w = re.findall('([\w\d-]+)\.s\.(\d+)\.w\.(\d+)',id)[0]
                if s != prevs and sentence:
                    yield sentence_id, sentence
                    sentence = []
                    sentence_id = doc_id + '.s.' + str(s)
            sentence.append( (word,id,pos,lemma) )
            prevs = s
        if sentence:
            yield sentence_id, sentence
158532
def _word_to_score(self, ids, scores):
        
        # should be 1-D vectors
        assert len(ids.shape) == 1
        assert ids.shape == scores.shape

        w2s = {}
        for i in range(len(ids)):
            w2s[self.vocab.index2word(ids[i])] = scores[i]
        return w2s
684548
def compare_report_print(sorted_list, scores, best_name):
    
    title_items = ["Rank", "Name", "Class-Score", "Overall-Score"]
    class_scores_len = map(lambda x: len(
        str(x["class"])), list(scores.values()))
    shifts = ["%-" +
              str(len(sorted_list) +
                  4) +
              "s", "%-" +
              str(max(map(lambda x: len(str(x)), sorted_list)) +
                  4) +
              "s", "%-" +
              str(max(class_scores_len) + 11) + "s"]
    result = ""
    result += "Best : " + str(best_name) + "\n\n"
    result += ("".join(shifts)
               ) % tuple(title_items[:-1]) + title_items[-1] + "\n"
    prev_rank = 0
    for index, cm in enumerate(sorted_list):
        rank = index
        if scores[sorted_list[rank]] == scores[sorted_list[prev_rank]]:
            rank = prev_rank
        result += ("".join(shifts)) % (str(rank + 1), str(cm),
                                       str(scores[cm]["class"])) + str(scores[cm]["overall"]) + "\n"
        prev_rank = rank
    if best_name is None:
        result += "\nWarning: " + COMPARE_RESULT_WARNING
    return result
440594
def scorerange(self, start, end, start_inclusive=True, end_inclusive=True):
        
        if not self:
            return []

        left = bisect_left(self._scores, (start,))
        right = bisect_right(self._scores, (end,))

        if end_inclusive:
            # end is inclusive
            while right < len(self) and self._scores[right][0] == end:
                right += 1

        if not start_inclusive:
            while left < right and self._scores[left][0] == start:
                left += 1
        return self._scores[left:right]
200718
public static final <T> Ordering<Scored<T>> ByScoreOnly() {
    return Ordering.natural().onResultOf(Scoreds.<T>scoresOnly());
  }
1245027
def pad_sentences(sentences, padding_word="</s>"):
    
    sequence_length = max(len(x) for x in sentences)
    padded_sentences = []
    for i, sentence in enumerate(sentences):
        num_padding = sequence_length - len(sentence)
        new_sentence = sentence + [padding_word] * num_padding
        padded_sentences.append(new_sentence)
    return padded_sentences
379906
def get_hints(code_list, k=10, hint_folder=HINT_FOLDER, current_tokens=None):
    

    def hint_score(v, size):
        """
        The formula for hint score
        """
        return 1.0 - abs(v / (size + 1) - 0.5)

    if len(code_list) <= 1:
        return [], []

    if current_tokens is None:
        current_tokens = []

    size = min(len(code_list), MAX_HINT_SMAPLING_SIZE)
    sample = random.sample(code_list, size)
    hint_list = []
    capital_dict = {}

    for code in sample:
        path = gen_path(hint_folder, code)
        fp = os.path.join(path, code)
        try:
            with open(fp) as f:
                hints = set(f.read().strip().split('\n'))
                hint_list.extend([h.lower() for h in hints])
                capital_dict.update({hint.lower(): hint for hint in hints})
        except FileNotFoundError:
            logging.warning("FileNotFoundError: No such file: %r" % fp )
    document_freq = Counter(hint_list)
    score = [(capital_dict[k], hint_score(v, size)) \
             for k, v in document_freq.items() if k not in current_tokens]
    if len(score) == 0:
        return [], []
    score.sort(key=lambda x: x[1], reverse=True)
    hints, scores = tuple(list(zip(*score[:k])))
    return hints, scores
122011
def _words_by_score(words, score, least_to_most, n=None):
    
    if words.shape != score.shape:
        raise ValueError('`words` and `score` must have the same shape')

    if n is not None and (n <= 0 or n > len(words)):
        raise ValueError('`n` must be in range [0, len(words)]')

    indices = np.argsort(score)
    if not least_to_most:
        indices = indices[::-1]

    ordered_words = words[indices]

    if n is not None:
        return ordered_words[:n]
    else:
        return ordered_words
361832
def _title_similarity_score(full_text, title):
    """"""

    sentences = sentence_tokenizer(full_text)
    norm = _normalize([title]+sentences)
    similarity_matrix = pairwise_kernels(norm, metric='cosine')
    return sorted(zip(
                     similarity_matrix[0,1:],
                     range(len(similarity_matrix)),
                     sentences
                     ),
                 key = lambda tup: tup[0],
                 reverse=True
                 )
1139547
def get_sentences_list(self, sentences=1):
        
        if sentences < 1:
            raise ValueError('Param "sentences" must be greater than 0.')

        sentences_list = []

        while sentences:
            num_rand_words = random.randint(self.MIN_WORDS, self.MAX_WORDS)

            random_sentence = self.make_sentence(
                random.sample(self.words, num_rand_words))

            sentences_list.append(random_sentence)
            sentences -= 1

        return sentences_list
904144
def _generate_out_list(results):
    '''
    
    '''
    passed = 0
    failed = 0
    skipped = 0
    missing_tests = 0
    total_time = 0.0
    for state in results:
        if not results[state].items():
            missing_tests = missing_tests + 1
        else:
            for dummy, val in results[state].items():
                log.info("dummy=%s, val=%s", dummy, val)
                if val['status'].startswith('Pass'):
                    passed = passed + 1
                if val['status'].startswith('Fail'):
                    failed = failed + 1
                if val['status'].startswith('Skip'):
                    skipped = skipped + 1
                total_time = total_time + float(val['duration'])
    out_list = []
    for key, value in results.items():
        out_list.append({key: value})
    out_list.sort()
    out_list.append({'TEST RESULTS': {'Execution Time': round(total_time, 4),
                                      'Passed': passed, 'Failed': failed, 'Skipped': skipped,
                                      'Missing Tests': missing_tests}})
    return out_list
184912
def results(self):
        

        results = {
            'overall': self.results_overall_metrics(),
            'class_wise': self.results_class_wise_metrics(),
            'class_wise_average': self.results_class_wise_average_metrics()
        }

        return results
140952
def _create_sentence_objects(self):
        """"""
        sentence_objects = []
        sentences = sent_tokenize(self.raw, tokenizer=self.tokenizer)
        char_index = 0  # Keeps track of character index within the blob
        for sent in sentences:
            # Compute the start and end indices of the sentence
            # within the blob. This only works if the sentence splitter
            # does not perform any character replacements or changes to
            # white space.
            # Working: NLTKPunktTokenizer
            # Not working: PatternTokenizer
            try:
                start_index = self.raw.index(sent, char_index)
                char_index += len(sent)
                end_index = start_index + len(sent)
            except ValueError:
                start_index = None
                end_index = None
            # Sentences share the same models as their parent blob
            s = Sentence(
                sent,
                start_index=start_index,
                end_index=end_index,
                tokenizer=self.tokenizer,
                np_extractor=self.np_extractor,
                pos_tagger=self.pos_tagger,
                analyzer=self.analyzer,
                parser=self.parser,
                classifier=self.classifier)
            sentence_objects.append(s)
        return sentence_objects
633609
def baldwinsoc_winners(self, profile):
        
        ordering = profile.getOrderVectors()
        m = profile.numCands
        prefcounts = profile.getPreferenceCounts()
        if min(ordering[0]) == 0:
            startstate = set(range(m))
        else:
            startstate = set(range(1, m + 1))
        wmg = self.getWmg2(prefcounts, ordering, startstate, normalize=False)
        known_winners = set()
        # ----------Some statistics--------------
        hashtable2 = set()

        # push the node of start state into the priority queue
        root = Node(value=startstate)
        stackNode = []
        stackNode.append(root)

        while stackNode:
            # ------------pop the current node-----------------
            node = stackNode.pop()
            # -------------------------------------------------
            state = node.value.copy()

            # goal state 1: if the state set contains only 1 candidate, then stop
            if len(state) == 1 and list(state)[0] not in known_winners:
                known_winners.add(list(state)[0])
                continue
            # goal state 2 (pruning): if the state set is subset of the known_winners set, then stop
            if state <= known_winners:
                continue
            # ----------Compute plurality score for the current remaining candidates--------------
            plural_score = dict()
            for cand in state:
                plural_score[cand] = 0
            for cand1, cand2 in itertools.permutations(state, 2):
                plural_score[cand1] += wmg[cand1][cand2]

            # if current state satisfies one of the 3 goal state, continue to the next loop

            # After using heuristics, generate children and push them into priority queue
            # frontier = [val for val in known_winners if val in state] + list(set(state) - set(known_winners))

            minscore = min(plural_score.values())
            for to_be_deleted in state:
                if plural_score[to_be_deleted] == minscore:
                    child_state = state.copy()
                    child_state.remove(to_be_deleted)
                    tpc = tuple(sorted(child_state))
                    if tpc in hashtable2:
                        continue
                    else:
                        hashtable2.add(tpc)
                        child_node = Node(value=child_state)
                        stackNode.append(child_node)
        return sorted(known_winners)
311015
def _get_keywords_with_score(extracted_lemmas, lemma_to_word):
    
    keywords = {}
    for score, lemma in extracted_lemmas:
        keyword_list = lemma_to_word[lemma]
        for keyword in keyword_list:
            keywords[keyword] = score
    return keywords
534408
def get_answer_begin_end(data):
    '''
    
    '''
    begin = []
    end = []
    for qa_pair in data:
        tokens = qa_pair['passage_tokens']
        char_begin = qa_pair['answer_begin']
        char_end = qa_pair['answer_end']
        word_begin = get_word_index(tokens, char_begin)
        word_end = get_word_index(tokens, char_end)
        begin.append(word_begin)
        end.append(word_end)
    return np.asarray(begin), np.asarray(end)