Load a list of words from which to generate a word frequency list

            Args:
                words (list): The list of words to be loaded
333356
def load_words(self, words):
        
        self._dictionary.update([word.lower() for word in words])
        self._update_dictionary()
1001920
def __get_keywords(self):
        
        txt = self.text
        for line in txt:
            for word in split_words(line):
                yield(word)
40545
def _init_from_list(self, subwords):
    """"""
    subwords = [tf.compat.as_text(s) for s in subwords if s]
    self._subwords = subwords
    # Note that internally everything is 0-indexed. Padding is dealt with at the
    # end of encode and the beginning of decode.
    self._subword_to_id = {s: i for i, s in enumerate(subwords)}

    # We remember the maximum length of any subword to avoid having to
    # check arbitrarily long strings.
    self._max_subword_len = max(
        len(_UNDERSCORE_REPLACEMENT), max([len(s) for s in subwords] or [1]))

    # Initialize the cache
    self._cache_size = 2**20
    self._token_to_ids_cache = [(None, None)] * self._cache_size

    # Setup tokenizer
    # Reserved tokens are all tokens that are mixed alphanum and non-alphanum.
    reserved_tokens = set([_UNDERSCORE_REPLACEMENT])
    for t in self._subwords:
      if text_encoder.is_mixed_alphanum(t):
        reserved_tokens.add(t)
    self._tokenizer = text_encoder.Tokenizer(
        alphanum_only=False, reserved_tokens=reserved_tokens)
636025
def train(self, word_s):
        
        for word in word_s:
            # 把词语的每个前缀更新到 prefix_set 中
            for index in range(len(word)):
                self._set.add(word[:index + 1])
914929
def from_vocabfile(cls, filename):
    
    words = [x.strip() for x in _open(filename, 'r').read().splitlines()]
    return cls(words=words)
1074641
def word_freqs() -> List[Tuple[str, int]]:
    
    lines = list(get_corpus(_FILENAME))
    listword = []
    for line in lines:
        listindata = line.split("\t")
        listword.append((listindata[0], int(listindata[1])))

    return listword
512059
def load_data_with_word2vec(word2vec_list):
    
    # Load and preprocess data
    sentences, labels = load_data_and_labels()
    sentences_padded = pad_sentences(sentences)
    # vocabulary, vocabulary_inv = build_vocab(sentences_padded)
    return build_input_data_with_word2vec(sentences_padded, labels, word2vec_list)
1330420
def _init_vocab_from_list(self, vocab_list):
    
    def token_gen():
      for token in vocab_list:
        if token not in RESERVED_TOKENS:
          yield token

    self._init_vocab(token_gen())
353630
def validate_words(word_list):
    '''
    
    '''
    if word_list is None:
        return {}
    elif isinstance(word_list, list):
        if not word_list:
            return {}
        else:
            return set(word for word in word_list if word in WORD_DISTRIBUTION)
    else:
        raise InputError("list variable not passed as argument to validate_words")
1204851
def words(self):
        
        return WordList(
            word_tokenize(self.raw, self.tokenizer, include_punc=False))
662251
public List<String> phraseDel(String[] words){
		List<String> list = new ArrayList<String>(); 
		String s;
		int length= words.length;
		for(int i = 0; i < length; i++){
			s = words[i];
			if(!isStopWord(s))
				list.add(s);
		}
		return list;
	}
546232
def tree_words(node):
    
    for word in split_text(tree_text(node)):
        word = word.strip()
        if word:
            yield word
965691
def get_or_generate_vocabulary(data_dir,
                               tmp_dir,
                               data_prefix,
                               max_page_size_exp,
                               approx_vocab_size=32768,
                               strip=True):
  
  num_pages_for_vocab_generation = approx_vocab_size // 3
  vocab_file = vocab_filename(approx_vocab_size, strip)

  def my_generator(data_prefix):
    """Line generator for vocab."""
    count = 0
    for page in corpus_page_generator(
        all_corpus_files(data_prefix)[::-1], tmp_dir, max_page_size_exp):
      revisions = page["revisions"]
      if revisions:
        text = get_text(revisions[-1], strip=strip)
        yield text
        count += 1
        if count % 100 == 0:
          tf.logging.info("reading pages for vocab %d" % count)
        if count > num_pages_for_vocab_generation:
          break

  return generator_utils.get_or_generate_vocab_inner(data_dir, vocab_file,
                                                     approx_vocab_size,
                                                     my_generator(data_prefix))
321676
def get_word_level_vocab(self):
        

        def simple_tokenize(source_str, token_delim=' ', seq_delim='\n'):
            return list(filter(None, re.split(token_delim + '|' + seq_delim, source_str)))

        return VocabProvider._create_squad_vocab(simple_tokenize, self._dataset)
129992
def load(fin, dtype=np.float32, max_vocab=None):
    
    vocab = {}
    line = next(fin)
    data = line.strip().split(b' ')
    assert len(data) == 2
    words = int(data[0])
    if max_vocab is not None:
        words = min(max_vocab, words)
    size = int(data[1])
    arr = np.empty((words, size), dtype=dtype)
    i = 0
    for n_line, line in enumerate(fin):
        if i >= words:
            break
        token, v = _load_line(line, dtype, size)
        if token in vocab:
            parse_warn(b'Duplicated vocabulary ' + token)
            continue
        arr[i, :] = v
        vocab[token] = i
        i += 1
    if i != words:
        # Use + instead of formatting because python 3.4.* does not allow
        # format with bytes
        parse_warn(
            b'EOF before the defined size (read ' + bytes(i) + b', expected '
            + bytes(words) + b')'
        )
        arr = arr[:i, :]
    return arr, vocab
1048611
def words(fpth):
    
    from collections import Counter
    words__ = Counter()
    with open(fpth) as fp:
        for line in fp:
            words__.update(line.strip().split(' '))
    return words__
1203829
def to_entropy(words)
      word_master = load_words
      mnemonic = words.map do |w|
        index = word_master.index(w.downcase)
        raise IndexError, 'word not found in words list.' unless index
        index.to_s(2).rjust(11, '0')
      end.join
      entropy = mnemonic.slice(0, (mnemonic.length * 32) / 33)
      checksum = mnemonic.gsub(entropy, '')
      raise SecurityError, 'checksum mismatch.' unless checksum == checksum(entropy)
      [entropy].pack('B*').bth
    end