Return words in `vocab` ordered by saliency score.
183453
def _words_by_salience_score(vocab, topic_word_distrib, doc_topic_distrib, doc_lengths, n=None, least_to_most=False):
    """"""
    saliency = get_word_saliency(topic_word_distrib, doc_topic_distrib, doc_lengths)
    return _words_by_score(vocab, saliency, least_to_most=least_to_most, n=n)
311015
def _get_keywords_with_score(extracted_lemmas, lemma_to_word):
    
    keywords = {}
    for score, lemma in extracted_lemmas:
        keyword_list = lemma_to_word[lemma]
        for keyword in keyword_list:
            keywords[keyword] = score
    return keywords
1324045
def read_vocab(args, column_name):
  
  vocab_path = os.path.join(args.analysis,
                            feature_transforms.VOCAB_ANALYSIS_FILE % column_name)

  if not file_io.file_exists(vocab_path):
    return []

  vocab, _ = feature_transforms.read_vocab_file(vocab_path)
  return vocab
1055653
def get_known_topics(self, lang):
        """"""
        return [topic['title']
                for topic in self.user_data.language_data[lang]['skills']
                if topic['learned']]
960614
def classify(self, text):
        
        score = self.score(text)
        if not score:
            return None
        return sorted(score.items(), key=lambda v: v[1])[-1][0]
545751
def scores(self, text: str) -> Dict[str, float]:
        
        values = extract(text)
        input_fn = _to_func(([values], []))
        prediction = self._classifier.predict_proba(input_fn=input_fn)
        probabilities = next(prediction).tolist()
        sorted_languages = sorted(self.languages)
        return dict(zip(sorted_languages, probabilities))
231302
def spelling_suggestions(self):
        """"""
        if not self.is_tagged(WORDS):
            self.tokenize_words()
        return [data[SUGGESTIONS] for data in vabamorf.spellcheck(self.word_texts, suggestions=True)]
965691
def get_or_generate_vocabulary(data_dir,
                               tmp_dir,
                               data_prefix,
                               max_page_size_exp,
                               approx_vocab_size=32768,
                               strip=True):
  
  num_pages_for_vocab_generation = approx_vocab_size // 3
  vocab_file = vocab_filename(approx_vocab_size, strip)

  def my_generator(data_prefix):
    """Line generator for vocab."""
    count = 0
    for page in corpus_page_generator(
        all_corpus_files(data_prefix)[::-1], tmp_dir, max_page_size_exp):
      revisions = page["revisions"]
      if revisions:
        text = get_text(revisions[-1], strip=strip)
        yield text
        count += 1
        if count % 100 == 0:
          tf.logging.info("reading pages for vocab %d" % count)
        if count > num_pages_for_vocab_generation:
          break

  return generator_utils.get_or_generate_vocab_inner(data_dir, vocab_file,
                                                     approx_vocab_size,
                                                     my_generator(data_prefix))
855757
def spellcheck_results(self):
        """"""
        if not self.is_tagged(WORDS):
            self.tokenize_words()
        return vabamorf.spellcheck(self.word_texts, suggestions=True)
798228
def vocab_counter(args):
  """"""
  if isinstance(args.input, TextFiles):
    v = CountedVocabulary.from_textfiles(args.input, workers=args.workers)
  else:
    v = CountedVocabulary.from_textfile(args.input, workers=args.workers)
  if args.min_count > 1: 
    v = v.min_count(args.min_count)
  if args.most_freq > 0:
    v = v.most_frequent(args.most_freq)
  print(v)
161587
def words(self, index = None):
        
        if index is None:
            return self.select(Word,None,True,default_ignore_structure)
        else:
            if index < 0:
                index = self.count(Word,None,True,default_ignore_structure) + index
            for i, e in enumerate(self.select(Word,None,True,default_ignore_structure)):
                if i == index:
                    return e
            raise IndexError
129992
def load(fin, dtype=np.float32, max_vocab=None):
    
    vocab = {}
    line = next(fin)
    data = line.strip().split(b' ')
    assert len(data) == 2
    words = int(data[0])
    if max_vocab is not None:
        words = min(max_vocab, words)
    size = int(data[1])
    arr = np.empty((words, size), dtype=dtype)
    i = 0
    for n_line, line in enumerate(fin):
        if i >= words:
            break
        token, v = _load_line(line, dtype, size)
        if token in vocab:
            parse_warn(b'Duplicated vocabulary ' + token)
            continue
        arr[i, :] = v
        vocab[token] = i
        i += 1
    if i != words:
        # Use + instead of formatting because python 3.4.* does not allow
        # format with bytes
        parse_warn(
            b'EOF before the defined size (read ' + bytes(i) + b', expected '
            + bytes(words) + b')'
        )
        arr = arr[:i, :]
    return arr, vocab
871555
def convert_vocab(vocab_file):
    """"""
    original_vocab = load_vocab(vocab_file)
    token_to_idx = dict(original_vocab)
    num_tokens = len(token_to_idx)
    idx_to_token = [None] * len(original_vocab)
    for word in original_vocab:
        idx = int(original_vocab[word])
        idx_to_token[idx] = word

    def swap(token, target_idx, token_to_idx, idx_to_token, swap_idx):
        original_idx = token_to_idx[token]
        original_token = idx_to_token[target_idx]
        token_to_idx[token] = target_idx
        token_to_idx[original_token] = original_idx
        idx_to_token[target_idx] = token
        idx_to_token[original_idx] = original_token
        swap_idx.append((original_idx, target_idx))

    reserved_tokens = [gluonnlp.vocab.BERTVocab.PADDING_TOKEN, gluonnlp.vocab.BERTVocab.CLS_TOKEN,
                       gluonnlp.vocab.BERTVocab.SEP_TOKEN, gluonnlp.vocab.BERTVocab.MASK_TOKEN]

    unknown_token = gluonnlp.vocab.BERTVocab.UNKNOWN_TOKEN
    padding_token = gluonnlp.vocab.BERTVocab.PADDING_TOKEN
    swap_idx = []
    assert unknown_token in token_to_idx
    assert padding_token in token_to_idx
    swap(unknown_token, 0, token_to_idx, idx_to_token, swap_idx)
    for i, token in enumerate(reserved_tokens):
        swap(token, i + 1, token_to_idx, idx_to_token, swap_idx)

    # sanity checks
    assert len(token_to_idx) == num_tokens
    assert len(idx_to_token) == num_tokens
    assert None not in idx_to_token
    assert len(set(idx_to_token)) == num_tokens

    bert_vocab_dict = {}
    bert_vocab_dict['idx_to_token'] = idx_to_token
    bert_vocab_dict['token_to_idx'] = token_to_idx
    bert_vocab_dict['reserved_tokens'] = reserved_tokens
    bert_vocab_dict['unknown_token'] = unknown_token
    bert_vocab_dict['padding_token'] = padding_token
    bert_vocab_dict['bos_token'] = None
    bert_vocab_dict['eos_token'] = None
    bert_vocab_dict['mask_token'] = gluonnlp.vocab.BERTVocab.MASK_TOKEN
    bert_vocab_dict['sep_token'] = gluonnlp.vocab.BERTVocab.SEP_TOKEN
    bert_vocab_dict['cls_token'] = gluonnlp.vocab.BERTVocab.CLS_TOKEN
    json_str = json.dumps(bert_vocab_dict)
    converted_vocab = gluonnlp.vocab.BERTVocab.from_json(json_str)
    return converted_vocab, swap_idx
1148665
def open_vocab_values(instance):
    
    if instance['type'] not in enums.VOCAB_PROPERTIES:
        return

    properties = enums.VOCAB_PROPERTIES[instance['type']]
    for prop in properties:
        if prop in instance:

            if type(instance[prop]) is list:
                values = instance[prop]
            else:
                values = [instance[prop]]

            for v in values:
                if not v.islower() or '_' in v or ' ' in v:
                    yield JSONError("Open vocabulary value '%s' should be all"
                                    " lowercase and use hyphens instead of"
                                    " spaces or underscores as word"
                                    " separators." % v, instance['id'],
                                    'open-vocab-format')
370887
def findOODWords(isleDict, wordList):
    '''
    
    '''
    oodList = []
    for word in wordList:
        try:
            isleDict.lookup(word)
        except WordNotInISLE:
            oodList.append(word)
                
    oodList = list(set(oodList))
    oodList.sort()
    
    return oodList
1111121
def get_or_create_vocab(self, data_dir, tmp_dir, force_get=False):
    """"""
    # We assume that vocab file is present in data_dir directory where the
    # data generated will be stored.
    vocab_filepath = os.path.join(data_dir, self.vocab_filename)
    encoder = text_encoder.SubwordTextEncoder(vocab_filepath)
    return encoder
1289422
def read_vocab_file(file_path):
  
  with file_io.FileIO(file_path, 'r') as f:
    vocab_pd = pd.read_csv(
        f,
        header=None,
        names=['vocab', 'count'],
        dtype=str,  # Prevent pd from converting numerical categories.
        na_filter=False)  # Prevent pd from converting 'NA' to a NaN.

  vocab = vocab_pd['vocab'].tolist()
  ex_count = vocab_pd['count'].astype(int).tolist()

  return vocab, ex_count