For each word in the vocab of `dtm` (i.e. its columns), return how often it occurs at least `min_val` times.
    If `proportions` is True, return proportions scaled to the number of documents instead of absolute numbers.
190250
def get_doc_frequencies(dtm, min_val=1, proportions=False):
    
    if dtm.ndim != 2:
        raise ValueError('`dtm` must be a 2D array/matrix')

    doc_freq = np.sum(dtm >= min_val, axis=0)

    if doc_freq.ndim != 1:
        doc_freq = doc_freq.A.flatten()

    if proportions:
        return doc_freq / dtm.shape[0]
    else:
        return doc_freq
1097044
def _keep(
    word_freq: int,
    min_freq: int,
    min_len: int,
    max_len: int,
    dict_filter: Callable[[str], bool],
):
    
    if not word_freq or word_freq[1] < min_freq:
        return False

    word = word_freq[0]
    if not word or len(word) < min_len or len(word) > max_len or word[0] == ".":
        return False

    return dict_filter(word)
622850
def normalize_columns(features):
    
    # Calculate inverse document frequency.
    features = features.tocsc()
    for j in range(features.shape[1]):
        document_frequency = features.getcol(j).data.size
        if document_frequency > 1:
            features.data[features.indptr[j]: features.indptr[j + 1]] =\
                features.data[features.indptr[j]: features.indptr[j + 1]]/np.sqrt(np.log(document_frequency))

    features = features.tocsr()

    return features
1111648
private void statistics(String word, int times, Map<String, AtomicInteger> container){
        container.putIfAbsent(word, new AtomicInteger());
        container.get(word).addAndGet(times);
    }
655829
def build_evenly_discretised_mfd(mfd):
    
    occur_rates = Node("occurRates", text=mfd.occurrence_rates)
    return Node("incrementalMFD",
                {"binWidth": mfd.bin_width, "minMag": mfd.min_mag},
                nodes=[occur_rates])
66092
def dbscan(self, eps=0.75, min_samples=3):
        
        est = DBSCAN(metric='precomputed', eps=eps, min_samples=min_samples)
        est.fit(self.get_dm(False))
        return Partition(est.labels_)
768445
def describe(self):
        
        counts = self.value_counts(dropna=False)
        freqs = counts / float(counts.sum())

        from pandas.core.reshape.concat import concat
        result = concat([counts, freqs], axis=1)
        result.columns = ['counts', 'freqs']
        result.index.name = 'categories'

        return result
948032
def min_count(self, n=1):
    
    word_count = {w:c for w,c in iteritems(self.word_count) if c >= n}
    return CountedVocabulary(word_count=word_count)
977759
def count_token_occurrences(cls, words):
        
        counts = {}
        for word in words:
            if word in counts:
                counts[word] += 1
            else:
                counts[word] = 1
        return counts
40827
public void fitOnTexts(String[] texts) {
        String[] sequence;
        for (String text : texts) {
            if (documentCount == null)
                documentCount = 1;
            else
                documentCount += 1;
            if (charLevel) {
                if (lower)
                    text = text.toLowerCase();
                sequence = text.split("");
            } else {
                sequence = textToWordSequence(text, filters, lower, split);
            }
            for (String word : sequence) {
                if (wordCounts.containsKey(word))
                    wordCounts.put(word, wordCounts.get(word) + 1);
                else
                    wordCounts.put(word, 1);
            }
            Set<String> sequenceSet = new HashSet<>(Arrays.asList(sequence));
            for (String word: sequenceSet) {
                if (wordDocs.containsKey(word))
                    wordDocs.put(word, wordDocs.get(word) + 1);
                else
                    wordDocs.put(word, 1);
            }
        }
        Map<String, Integer> sortedWordCounts = reverseSortByValues((HashMap) wordCounts);

        ArrayList<String> sortedVocabulary = new ArrayList<>();
        if (outOfVocabularyToken != null)
            sortedVocabulary.add(outOfVocabularyToken);
        for (String word: sortedWordCounts.keySet()) {
            sortedVocabulary.add(word);
        }

        for (int i = 0; i < sortedVocabulary.size(); i++)
            wordIndex.put(sortedVocabulary.get(i), i+1);

        for(String key : wordIndex.keySet()){
            indexWord.put(wordIndex.get(key), key);
        }

        for (String key: wordDocs.keySet())
            indexDocs.put(wordIndex.get(key), wordDocs.get(key));
    }
1325502
def compact(self, term_doc_matrix):
		'''
		
		'''
		tdf = self.term_ranker(term_doc_matrix).get_ranks()
		tdf_sum = tdf.sum(axis=0)
		tdf_portions = tdf / tdf_sum
		threshold = np.max(self.term_count / tdf_sum)
		terms_to_remove = tdf_portions[~(tdf_portions > threshold).any(axis=1)].index
		return term_doc_matrix.remove_terms(terms_to_remove)
1363852
def Eg(self, **kwargs):
        '''
        
        '''
        return min(self.Eg_Gamma(**kwargs),
                   self.Eg_L(**kwargs),
                   self.Eg_X(**kwargs))
707142
def _relative_frequency(self, word):
		""""""

		count = self.type_counts.get(word, 0)
		return math.log(count/len(self.type_counts)) if count > 0 else 0
658890
def get_tfidf(self, term, document, normalized=False):
        
        tf = self.get_term_frequency(term, document)

        # Speeds up performance by avoiding extra calculations
        if tf != 0.0:
            # Add 1 to document frequency to prevent divide by 0
            # (Laplacian Correction)
            df = 1 + self.get_document_frequency(term)
            n = 2 + len(self._documents)

            if normalized:
                tf /= self.get_document_length(document)

            return tf * math.log10(n / df)
        else:
            return 0.0
635847
public static int word_count(EvaluationContext ctx, Object text, @BooleanDefault(false) Object bySpaces) {
        String _text = Conversions.toString(text, ctx);
        boolean _bySpaces = Conversions.toBoolean(bySpaces, ctx);
        return getWords(_text, _bySpaces).size();
    }
63110
def get_scores_for_category(self, cat_word_counts, not_cat_word_counts):
		'''
		
		'''
		beta = self.beta_
		assert len(cat_word_counts) == len(not_cat_word_counts)
		if self.priors_ is not None:
			p = self.priors_
			assert len(p) == len(cat_word_counts)
			precision = ((cat_word_counts + p * 1.) /
			             (cat_word_counts + not_cat_word_counts + 2 * p))
			recall = (cat_word_counts + p) * 1. / (cat_word_counts.sum() + p.sum())
		else:
			precision = (cat_word_counts * 1. / (cat_word_counts + not_cat_word_counts))
			recall = cat_word_counts * 1. / cat_word_counts.sum()
		precision_normcdf = ScaledFScore._safe_scaler(self.scaler_algo_, precision)
		recall_normcdf = ScaledFScore._safe_scaler(self.scaler_algo_, recall)
		scores = self._weighted_h_mean(precision_normcdf, recall_normcdf)
		scores[np.isnan(scores)] = 0.
		return scores
36846
def phrases_reduce(key, values):
  """"""
  if len(values) < 10:
    return
  counts = {}
  for filename in values:
    counts[filename] = counts.get(filename, 0) + 1

  words = re.sub(r":", " ", key)
  threshold = len(values) / 2
  for filename, count in counts.items():
    if count > threshold:
      yield "%s:%s\n" % (words, filename)