Returns a Hash of frequencies of each uniq word in the text
970984
def frequencies_for(words)
      word_frequencies = {}
      words.each { |word| word_frequencies[word] = words.count(word) }
      word_frequencies
    end
426104
def distinct_words(string_matrix: List[List[str]]) -> Set[str]:
    
    return set([word
                for sentence in string_matrix
                for word in sentence])
1033960
def compute_ngrams(word_list, S=3, T=3):
    
    _ngrams = []
    if isinstance(word_list, str):
        word_list = [word_list]
    for word in word_list:
        for n in range(S, T+1):
            _ngrams += zip(*(word[i:] for i in range(n)))
    return [''.join(_ngram) for _ngram in _ngrams]
861855
def other_seqids(self):
        """"""
        seqids = self._xml_root.xpath('GBSeq_other-seqids/GBSeqid/text()')
        return {t: l.rstrip('|').split('|')
                for t, _, l in [si.partition('|') for si in seqids]}
1035287
def analyze_chunks(data, freq, bits, chunksize=512):
    
    res = []
    while data != []:
        f = find_frequencies(data[:chunksize], freq, bits)
        res.append(sorted(find_notes(f), key=operator.itemgetter(1))[-1][0])
        data = data[chunksize:]
    return res
752568
def word_texts(self):
        """"""
        if not self.is_tagged(WORDS):
            self.tokenize_words()
        return [word[TEXT] for word in self[WORDS]]
319742
def use_general_term_frequencies(self):
		'''
		
		'''
		tdf = self._get_relevant_term_freq()
		bg_df = self.term_doc_mat.get_term_and_background_counts()[['background']]
		bg_df = pd.merge(tdf,
		                 bg_df,
		                 left_index=True,
		                 right_index=True,
		                 how='left').fillna(0.)
		self._store_priors_from_background_dataframe(bg_df)
		return self
702394
private int calcCountSubstringsWords(String str, HashSet<String> words) {
    int wordCount = 0;
    int N = str.length();
    for (int i = 0; i < N - 1; i++)
      for (int j = i + 2; j < N + 1; j++) {
        if (words.contains(str.substring(i, j)))
          wordCount += 1;
      }
    return wordCount;
  }
881013
def wordcount(text):
    ''''''
    bannedwords = read_file('stopwords.txt')
    wordcount = {}
    separated = separate(text)
    for word in separated:
        if word not in bannedwords:
            if not wordcount.has_key(word):
                wordcount[word] = 1
            else:
                wordcount[word] += 1
    return wordcount
1230486
def consistency_check(text, word_pairs, err, msg, offset=0):
    """"""
    errors = []

    msg = " ".join(msg.split())

    for w in word_pairs:
        matches = [
            [m for m in re.finditer(w[0], text)],
            [m for m in re.finditer(w[1], text)]
        ]

        if len(matches[0]) > 0 and len(matches[1]) > 0:

            idx_minority = len(matches[0]) > len(matches[1])

            for m in matches[idx_minority]:
                errors.append((
                    m.start() + offset,
                    m.end() + offset,
                    err,
                    msg.format(w[~idx_minority], m.group(0)),
                    w[~idx_minority]))

    return errors
221858
def distances(text, classnames=nil)
      classnames ||= @classnames
      distances = Hash.new 0

      th = TextHash.new(text)
      th.each { |word, count|
        thprob = count.to_f / th.length.to_f
        probs = get_word_probs(word, classnames)
        classnames.each { |k| 
          distances[k] += (thprob * Math.log(thprob / probs[k]) * count) 
        }
      }

      distances
    end
1081432
function cloneMatches(words) {
  var result = {}
  var key
  var match

  for (key in words) {
    match = words[key]

    result[key] = {
      matches: match.matches,
      stem: match.stem,
      score: match.score
    }
  }

  return result
}
1234383
def most_frequent(self, k):
    
    word_count = {w:self.word_count[w] for w in self.words[:k]}
    return CountedVocabulary(word_count=word_count)
556505
def eliminate_repeats(text):
    ''''''
    
    bannedwords = read_file('stopwords.txt')
    alphabet = 'abcdefghijklmnopqrstuvwxyz'
    words = text.split()
    standardwords = []
    for word in words:
        newstr = ''
        for char in word:
            if char in alphabet or char in alphabet.upper():
               newstr += char
        if newstr not in standardwords and newstr != '' and newstr not in bannedwords:
            standardwords.append(newstr)
        
    return map(lambda x: x.lower(),standardwords)
456064
def _get_sequences(self, *sequences):
        
        # by words
        if not self.qval:
            return [s.split() for s in sequences]
        # by chars
        if self.qval == 1:
            return sequences
        # by n-grams
        return [find_ngrams(s, self.qval) for s in sequences]
798228
def vocab_counter(args):
  """"""
  if isinstance(args.input, TextFiles):
    v = CountedVocabulary.from_textfiles(args.input, workers=args.workers)
  else:
    v = CountedVocabulary.from_textfile(args.input, workers=args.workers)
  if args.min_count > 1: 
    v = v.min_count(args.min_count)
  if args.most_freq > 0:
    v = v.most_frequent(args.most_freq)
  print(v)
1188522
def interpret_weekly_variant
      hash_for_occ_base = { type: :weekly, interval: interval }
      array_of_occurrences = []
      repeats_on.each do |day_of_week|
        array_of_occurrences << hash_for_occ_base.merge(day_of_week: day_of_week)
      end
      array_of_occurrences
    end