Provides word level vocabulary

        Returns
        -------
        Vocab
            Word level vocabulary
321676
def get_word_level_vocab(self):
        

        def simple_tokenize(source_str, token_delim=' ', seq_delim='\n'):
            return list(filter(None, re.split(token_delim + '|' + seq_delim, source_str)))

        return VocabProvider._create_squad_vocab(simple_tokenize, self._dataset)
1045264
def get_tokens(max_value):
  
  vocab = [str(i) for i in range(max_value)]
  vocab = set(vocab)
  vocab.update(CodeOp.LITERALS)
  vocab.update(CodeOp.KEYWORDS)
  vocab |= set("".join(vocab))
  return sorted(vocab)
285860
def get_docs(self, vocab):
        """"""
        for string in self.strings:
            vocab[string]
        orth_col = self.attrs.index(ORTH)
        for tokens, spaces in zip(self.tokens, self.spaces):
            words = [vocab.strings[orth] for orth in tokens[:, orth_col]]
            doc = Doc(vocab, words=words, spaces=spaces)
            doc = doc.from_array(self.attrs, tokens)
            yield doc
871555
def convert_vocab(vocab_file):
    """"""
    original_vocab = load_vocab(vocab_file)
    token_to_idx = dict(original_vocab)
    num_tokens = len(token_to_idx)
    idx_to_token = [None] * len(original_vocab)
    for word in original_vocab:
        idx = int(original_vocab[word])
        idx_to_token[idx] = word

    def swap(token, target_idx, token_to_idx, idx_to_token, swap_idx):
        original_idx = token_to_idx[token]
        original_token = idx_to_token[target_idx]
        token_to_idx[token] = target_idx
        token_to_idx[original_token] = original_idx
        idx_to_token[target_idx] = token
        idx_to_token[original_idx] = original_token
        swap_idx.append((original_idx, target_idx))

    reserved_tokens = [gluonnlp.vocab.BERTVocab.PADDING_TOKEN, gluonnlp.vocab.BERTVocab.CLS_TOKEN,
                       gluonnlp.vocab.BERTVocab.SEP_TOKEN, gluonnlp.vocab.BERTVocab.MASK_TOKEN]

    unknown_token = gluonnlp.vocab.BERTVocab.UNKNOWN_TOKEN
    padding_token = gluonnlp.vocab.BERTVocab.PADDING_TOKEN
    swap_idx = []
    assert unknown_token in token_to_idx
    assert padding_token in token_to_idx
    swap(unknown_token, 0, token_to_idx, idx_to_token, swap_idx)
    for i, token in enumerate(reserved_tokens):
        swap(token, i + 1, token_to_idx, idx_to_token, swap_idx)

    # sanity checks
    assert len(token_to_idx) == num_tokens
    assert len(idx_to_token) == num_tokens
    assert None not in idx_to_token
    assert len(set(idx_to_token)) == num_tokens

    bert_vocab_dict = {}
    bert_vocab_dict['idx_to_token'] = idx_to_token
    bert_vocab_dict['token_to_idx'] = token_to_idx
    bert_vocab_dict['reserved_tokens'] = reserved_tokens
    bert_vocab_dict['unknown_token'] = unknown_token
    bert_vocab_dict['padding_token'] = padding_token
    bert_vocab_dict['bos_token'] = None
    bert_vocab_dict['eos_token'] = None
    bert_vocab_dict['mask_token'] = gluonnlp.vocab.BERTVocab.MASK_TOKEN
    bert_vocab_dict['sep_token'] = gluonnlp.vocab.BERTVocab.SEP_TOKEN
    bert_vocab_dict['cls_token'] = gluonnlp.vocab.BERTVocab.CLS_TOKEN
    json_str = json.dumps(bert_vocab_dict)
    converted_vocab = gluonnlp.vocab.BERTVocab.from_json(json_str)
    return converted_vocab, swap_idx
965691
def get_or_generate_vocabulary(data_dir,
                               tmp_dir,
                               data_prefix,
                               max_page_size_exp,
                               approx_vocab_size=32768,
                               strip=True):
  
  num_pages_for_vocab_generation = approx_vocab_size // 3
  vocab_file = vocab_filename(approx_vocab_size, strip)

  def my_generator(data_prefix):
    """Line generator for vocab."""
    count = 0
    for page in corpus_page_generator(
        all_corpus_files(data_prefix)[::-1], tmp_dir, max_page_size_exp):
      revisions = page["revisions"]
      if revisions:
        text = get_text(revisions[-1], strip=strip)
        yield text
        count += 1
        if count % 100 == 0:
          tf.logging.info("reading pages for vocab %d" % count)
        if count > num_pages_for_vocab_generation:
          break

  return generator_utils.get_or_generate_vocab_inner(data_dir, vocab_file,
                                                     approx_vocab_size,
                                                     my_generator(data_prefix))
908252
@SuppressWarnings("unchecked") // method is safe, since all calls inside are using generic SequenceElement methods
    public VocabCache<T> buildMergedVocabulary(@NonNull WordVectors wordVectors, boolean fetchLabels) {
        return buildMergedVocabulary((VocabCache<T>) wordVectors.vocab(), fetchLabels);
    }
876331
def _get_token_encoder(vocab_dir, vocab_name, filename):
  """"""
  vocab_path = os.path.join(vocab_dir, vocab_name)
  if not tf.gfile.Exists(vocab_path):
    _build_vocab(filename, vocab_path, 10000)
  return text_encoder.TokenTextEncoder(vocab_path)
914576
def resolver(self, vocab_data, attribute):
        
        term_list = vocab_data.get(self.content_vocab, [])
        # Loop through the terms from the vocabulary.
        for term_dict in term_list:
            # Match the name to the current content.
            if term_dict['name'] == self.content:
                return term_dict[attribute]
        return self.content
378136
def load_common(self, wlist):
        """"""
        self.com_dict = os.path.join(self.base_dir, 'dicts', 'EN_vocab.txt')
        with open(self.com_dict) as words_file:
            data = words_file.read()
        self.common_words = set(data.splitlines())
        if wlist:
            new_words = ''
            for wl in wlist:
                with open(wl) as f:
                    new_words += f.read()
            if new_words:
                new_dict = set(data.splitlines())
                self.common_words.update(new_dict)
701538
def build_vocab(dataset):
    
    counter = nlp.data.count_tokens([w for e in dataset for s in e[:2] for w in s],
                                    to_lower=True)
    vocab = nlp.Vocab(counter)
    return vocab
1239622
def load_words(wordlist=None):
    
    global words
    if not wordlist:
        # no wordlist was provided, load the wordlist from the local store
        filename = get_data('wordlist.txt')
        f = open(filename)
        wordlist = f.readlines()
        wordlist = [w.strip() for w in wordlist if w]
    words = wordlist
652069
def load_target_vocab(folder: str) -> Vocab:
    
    return vocab_from_json(os.path.join(folder, C.VOCAB_TRG_NAME % 0))
158422
public function getTermsForVocabulary($vocabulary_id)
    {
        return $this->cache->remember(
            'Rocket::Taxonomy::Terms::' . $vocabulary_id,
            60,
            function () use ($vocabulary_id) {
                $terms = TermContainer::where('vocabulary_id', $vocabulary_id)->get(['id']);

                $results = [];
                if (!empty($terms)) {
                    foreach ($terms as $term) {
                        $results[] = $term->id;
                    }
                }

                return $results;
            }
        );
    }
1251833
def known(self, words: List[str]) -> List[str]:
        
        return list(w for w in words if w in self.__WORDS)
913327
def collect_vocab(qp_pairs):
    '''
    
    '''
    vocab = set()
    for qp_pair in qp_pairs:
        for word in qp_pair['question_tokens']:
            vocab.add(word['word'])
        for word in qp_pair['passage_tokens']:
            vocab.add(word['word'])
    return vocab
592119
def get_vocab(self, vocab_name, **kwargs):
        
        vocab_dict = self.__get_vocab_dict__(vocab_name, **kwargs)

        filepaths = list(set([os.path.join(self.cache_dir,
                                           vocab_dict['filename']),
                              os.path.join(self.vocab_dir,
                                           vocab_dict['filename'])]))
        for path in filepaths:
            if os.path.exists(path):
                with open(path, 'rb') as f_obj:
                    vocab_dict.update({"name": vocab_name,
                                       "data": f_obj.read(),
                                       "modified": os.path.getmtime(path)})
                return vocab_dict
        download_locs = make_list(vocab_dict.get('download',[]))
        for loc in download_locs:

            loc_web = urllib.request.urlopen(loc)
            # loc_file_date = date_parse(loc_web.info()['Last-Modified'])
            urllib.request.urlretrieve(loc, filepaths[0])
            with open(filepaths[0], 'rb') as f_obj:
                vocab_dict.update({"name": vocab_name,
                                   "data": f_obj.read(),
                                   "modified": os.path.getmtime(filepaths[0])})
                return vocab_dict
1030914
def vocab_from_json(path: str, encoding: str = C.VOCAB_ENCODING) -> Vocab:
    
    with open(path, encoding=encoding) as inp:
        vocab = json.load(inp)
        utils.check_condition(is_valid_vocab(vocab), "Vocabulary %s not valid." % path)
        logger.info('Vocabulary (%d words) loaded from "%s"', len(vocab), path)
        return vocab