Creates a key/value set of word/count for a given sample of text

        :param words: full list of all tokens, non-unique
        :type words: list
        :return: key/value pairs of words and their counts in the list
        :rtype: dict
977759
def count_token_occurrences(cls, words):
        
        counts = {}
        for word in words:
            if word in counts:
                counts[word] += 1
            else:
                counts[word] = 1
        return counts
1330400
def contains_text_tokens(self, text_tokens):
        
        id_run = []
        for token in text_tokens:
            id = self.word_list.get(token, None)
            if id is None:
                return False  # since a token is not in the word list, the phrase is not in the dictionary
            id_run.append(id)
        return self.contains_tokens(id_run)
301366
def words(total)
      if total.class == Range
        (1..interpret_value(total)).map { WORDS[random_number(total.min, total.max)] }.join(' ')
      else
        (1..interpret_value(total)).map { WORDS[random_number(0, total)] }.join(' ')
      end
    end
995099
def word_count(ctx, text, by_spaces=False):
    
    text = conversions.to_string(text, ctx)
    by_spaces = conversions.to_boolean(by_spaces, ctx)
    return len(__get_words(text, by_spaces))
886679
def word_counts(self):
    
    counts = defaultdict(int)
    for word in self.words:
        counts[word] += 1
    return counts
687177
def sample(path, start, length):
    
    # initialize vocabulary and sequence length
    param.seq_len = 1
    ds = CharRNNData(param.corpus, 100000)

    pred = OfflinePredictor(PredictConfig(
        model=Model(),
        session_init=SaverRestore(path),
        input_names=['input', 'c0', 'h0', 'c1', 'h1'],
        output_names=['prob', 'last_state']))

    # feed the starting sentence
    initial = np.zeros((1, param.rnn_size))
    for c in start[:-1]:
        x = np.array([[ds.char2idx[c]]], dtype='int32')
        _, state = pred(x, initial, initial, initial, initial)

    def pick(prob):
        t = np.cumsum(prob)
        s = np.sum(prob)
        return(int(np.searchsorted(t, np.random.rand(1) * s)))

    # generate more
    ret = start
    c = start[-1]
    for k in range(length):
        x = np.array([[ds.char2idx[c]]], dtype='int32')
        prob, state = pred(x, state[0, 0], state[0, 1], state[1, 0], state[1, 1])
        c = ds.chars[pick(prob[0])]
        ret += c
    print(ret)
333356
def load_words(self, words):
        
        self._dictionary.update([word.lower() for word in words])
        self._update_dictionary()
846248
def get_word_counts(word)
      # fetch all (class,count) pairs for a given word
      row = @cassandra.get(:classes, word.to_s)
      return row.to_hash if row.empty?
      row.inject({}){|counts, col| counts[col.first.to_sym] = [col.last.to_f,0].max; counts}
    end
1049760
function countWords (text, ParseTree, isWord) {
  isWord = isWord || {};
  function countWord (word) {
    if (word)
      isWord[word.toLowerCase()] = true;
  }
  function countWordsAtNodes (nodes) {
    if (nodes)
      nodes.forEach (countWordsAtNode);
  }
  function countWordsAtNode (node) {
    if (typeof(node) === 'object') {
      if (node.functag)
	countWord (ParseTree.funcChar + node.functag);
      switch (node.type) {
      case 'lookup':
	countWord (ParseTree.varChar + node.varname);
        break
      case 'assign':
	countWord (ParseTree.varChar + node.varname);
        countWordsAtNodes (node.value);
        countWordsAtNodes (node.local);
        break
      case 'alt':
        node.opts.forEach (countWordsAtNodes);
        break
      case 'rep':
        countWordsAtNodes (node.unit);
        break
      case 'func':
	countWord (ParseTree.funcChar + node.funcname);
	switch (node.funcname) {
	case 'eval':
          countWordsAtNodes (node.value);
          countWordsAtNodes (node.args);
	  break
	case 'strictquote':
	case 'quote':
	case 'unquote':
        default:
          countWordsAtNodes (node.args);
          break
	}
        break
      case 'cond':
        if (ParseTree.isTraceryExpr (node))
	  countWord (ParseTree.traceryChar + ParseTree.traceryVarName(node) + ParseTree.traceryChar);
        countWordsAtNodes (node.test);
        countWordsAtNodes (node.t);
        countWordsAtNodes (node.f);
        break
      case 'root':
      case 'alt_sampled':
        countWordsAtNodes (node.rhs);
        break
      case 'rep_sampled':
        node.reps.forEach (countWordsAtNodes);
        break
      case 'sym':
	countWord (ParseTree.symChar + ParseTree.leftBraceChar + (node.user || ParseTree.defaultUser) + '/' + node.name + ParseTree.rightBraceChar);
        countWordsAtNodes (node.rhs);
        countWordsAtNodes (node.bind);
        break
      default:
	break
      }
    } else if (typeof(node) === 'string')
      node
      .replace(/[^a-zA-Z0-9_]/g,' ')  // these are the characters we keep
      .replace(/\s+/g,' ').replace(/^ /,'').replace(/ $/,'')  // collapse all runs of space & remove start/end space
      .split(' ')
      .forEach (countWord);
  }
  
  var parsed = ParseTree.parseRhs (text);
  countWordsAtNodes (parsed);

  return isWord;
}
289737
def add_to_dict(self, text):
        
        n = self.n

        sentences = re.split(r'(?<!\w\.\w.)(?<![A-Z][a-z]\.)(?<=\.|\?|!)\s', text)
        # '' is a special symbol for the start of a sentence like pymarkovchain uses
        for sentence in sentences:
            sentence = sentence.replace('"','') # remove quotes
            words = sentence.strip().split()  # split each sentence into its constituent words
            if len(words) == 0:
                continue

            # first word follows a sentence end
            self.word_dict[("",)][words[0]].count += 1

            for j in range(1, n+1):
                for i in range(len(words) - 1):
                    if i + j >= len(words):
                        continue
                    word = tuple(words[i:i + j])
                    self.word_dict[word][words[i + j]].count += 1

                # last word precedes a sentence end
                self.word_dict[tuple(words[len(words) - j:len(words)])][""].count += 1

        # We've now got the db filled with parametrized word counts
        # We still need to normalize this to represent probabilities
        for word in self.word_dict:
            wordsum = 0
            for nextword in self.word_dict[word]:
                wordsum += self.word_dict[word][nextword].count
            if wordsum != 0:
                for nextword in self.word_dict[word]:
                    self.word_dict[word][nextword].prob = self.word_dict[word][nextword].count / wordsum
795392
def convert_examples_to_features(examples, label_list, max_seq_length,
                                 tokenizer, output_mode):
    """"""

    label_map = {label : i for i, label in enumerate(label_list)}

    features = []
    for (ex_index, example) in enumerate(examples):
        if ex_index % 10000 == 0:
            logger.info("Writing example %d of %d" % (ex_index, len(examples)))

        tokens_a = tokenizer.tokenize(example.text_a)

        tokens_b = None
        if example.text_b:
            tokens_b = tokenizer.tokenize(example.text_b)
            # Modifies `tokens_a` and `tokens_b` in place so that the total
            # length is less than the specified length.
            # Account for [CLS], [SEP], [SEP] with "- 3"
            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)
        else:
            # Account for [CLS] and [SEP] with "- 2"
            if len(tokens_a) > max_seq_length - 2:
                tokens_a = tokens_a[:(max_seq_length - 2)]

        # The convention in BERT is:
        # (a) For sequence pairs:
        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]
        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1
        # (b) For single sequences:
        #  tokens:   [CLS] the dog is hairy . [SEP]
        #  type_ids: 0   0   0   0  0     0 0
        #
        # Where "type_ids" are used to indicate whether this is the first
        # sequence or the second sequence. The embedding vectors for `type=0` and
        # `type=1` were learned during pre-training and are added to the wordpiece
        # embedding vector (and position vector). This is not *strictly* necessary
        # since the [SEP] token unambiguously separates the sequences, but it makes
        # it easier for the model to learn the concept of sequences.
        #
        # For classification tasks, the first vector (corresponding to [CLS]) is
        # used as as the "sentence vector". Note that this only makes sense because
        # the entire model is fine-tuned.
        tokens = ["[CLS]"] + tokens_a + ["[SEP]"]
        segment_ids = [0] * len(tokens)

        if tokens_b:
            tokens += tokens_b + ["[SEP]"]
            segment_ids += [1] * (len(tokens_b) + 1)

        input_ids = tokenizer.convert_tokens_to_ids(tokens)

        # The mask has 1 for real tokens and 0 for padding tokens. Only real
        # tokens are attended to.
        input_mask = [1] * len(input_ids)

        # Zero-pad up to the sequence length.
        padding = [0] * (max_seq_length - len(input_ids))
        input_ids += padding
        input_mask += padding
        segment_ids += padding

        assert len(input_ids) == max_seq_length
        assert len(input_mask) == max_seq_length
        assert len(segment_ids) == max_seq_length

        if output_mode == "classification":
            label_id = label_map[example.label]
        elif output_mode == "regression":
            label_id = float(example.label)
        else:
            raise KeyError(output_mode)

        if ex_index < 5:
            logger.info("*** Example ***")
            logger.info("guid: %s" % (example.guid))
            logger.info("tokens: %s" % " ".join(
                    [str(x) for x in tokens]))
            logger.info("input_ids: %s" % " ".join([str(x) for x in input_ids]))
            logger.info("input_mask: %s" % " ".join([str(x) for x in input_mask]))
            logger.info(
                    "segment_ids: %s" % " ".join([str(x) for x in segment_ids]))
            logger.info("label: %s (id = %d)" % (example.label, label_id))

        features.append(
                InputFeatures(input_ids=input_ids,
                              input_mask=input_mask,
                              segment_ids=segment_ids,
                              label_id=label_id))
    return features
531369
def keywords(text):
    
    text = split_words(text)
    numWords = len(text)  # of words before removing blacklist words
    freq = Counter(x for x in text if x not in stopWords)

    minSize = min(10, len(freq))  # get first 10
    keywords = {x: y for x, y in freq.most_common(minSize)}  # recreate a dict

    for k in keywords:
        articleScore = keywords[k]*1.0 / numWords
        keywords[k] = articleScore * 1.5 + 1

    return keywords
782844
def from_vocabfile(filename):
    
    word_count = [x.strip().split() for x in _open(filename, 'r').read().splitlines()]
    word_count = {w:int(c) for w,c in word_count}
    return CountedVocabulary(word_count=word_count)
707322
def find_tracked_words(self, tokens):
        
        tracked_words = []

        for i in range(len(tokens)):
            for j in range(i + 1, len(tokens) + 1):
                phrase = tokens[i:j]
                status = self.has_tokens(phrase)

                if status is not None:
                    if status is True:
                        tracked_words.append(TokenTrie.Token(phrase, i, j - 1))
                    elif status is False:
                        break

        return tracked_words
586338
def extract_keywords_from_text(self, text):
        
        sentences = nltk.tokenize.sent_tokenize(text)
        self.extract_keywords_from_sentences(sentences)
1228515
def create(cls, tokens:Tokens, max_vocab:int, min_freq:int) -> 'Vocab':
        ""
        freq = Counter(p for o in tokens for p in o)
        itos = [o for o,c in freq.most_common(max_vocab) if c >= min_freq]
        for o in reversed(defaults.text_spec_tok):
            if o in itos: itos.remove(o)
            itos.insert(0, o)
        return cls(itos)
306280
def word(ctx, text, number, by_spaces=False):
    
    return word_slice(ctx, text, number, conversions.to_integer(number, ctx) + 1, by_spaces)