Calculates the inverse document frequency for each
term. This is a representation of how often a certain
term is used in comparison to all terms.
324664
public function createInverseDocumentFrequency()
    {
        $documentCount = $this->getTotalDocumentCount();
        $inverseDocumentFrequency = new InverseDocumentFrequency($this->getSearchDbHandle(), $documentCount);
        $inverseDocumentFrequency->execute();
    }
236651
def compute_topic_document_probability
      outp = []

      @corpus.documents.each_with_index do |doc, idx|
        tops = [0.0] * num_topics
        ttl  = doc.counts.inject(0.0) { |sum, i| sum + i }
        phi[idx].each_with_index do |word_dist, word_idx|
          word_dist.each_with_index do |top_prob, top_idx|
            tops[top_idx] += Math.log(top_prob) * doc.counts[word_idx]
          end
        end
        tops = tops.map { |i| i / ttl }
        outp << tops
      end

      outp
    end
202990
def get_term_freq(self, go_id):
        '''
            
        '''
        num_ns = float(self.get_total_count(self.go2obj[go_id].namespace))
        return float(self.get_count(go_id))/num_ns if num_ns != 0 else 0
249506
def frequency_model( self ):
        
        prev_letter = None
        # use a generator in corpus
        prev_letter = list(self.corpus.next_tamil_letter())[0]
        for next_letter in self.corpus.next_tamil_letter():
            # update frequency from corpus
            key = prev_letter+next_letter
            val = self.bigram.get(key,None)
            prev_letter = next_letter
            if not val:
                self.bigram[key] = 0
            self.bigram[key] += 1
        
        return
901195
def _correct(token, term_freq):
    
    if token.lower() in term_freq:
        return token
    e1 = [t for t in _ed1(token) if t in term_freq]
    if len(e1) > 0:
        e1.sort(key=term_freq.get)
        return e1[0]
    e2 = [t for t in _ed2(token) if t in term_freq]
    if len(e2) > 0:
        e2.sort(key=term_freq.get)
        return e2[0]
    return token
861565
def match_query_to_corpus(self):
		
		ranking = []
		for i,doc in enumerate(self.processed_corpus):
			rank = 0.0
			for word in self.processed_query:
				if word in doc:
					rank += self.term_frequencies[i][word] * self.inverse_document_frequencies[word]
			ranking.append((rank,i))
		matching_corpus_index = 0
		max_rank = 0
		for rank,index in ranking:
			if rank > max_rank:
				matching_corpus_index = index
				max_rank = rank
		return matching_corpus_index
1304013
private void addTermFrequencies(Reader r, Map<String, Int> termFreqMap, String fieldName) throws IOException
   {
      TokenStream ts = analyzer.tokenStream(fieldName, r);
      int tokenCount = 0;
      // for every token
      while (ts.incrementToken())
      {
         CharTermAttribute term = ts.getAttribute(CharTermAttribute.class);
         String word = new String(term.buffer(), 0, term.length());
         tokenCount++;
         if (tokenCount > maxNumTokensParsed)
         {
            break;
         }
         if (isNoiseWord(word))
         {
            continue;
         }

         // increment frequency
         Int cnt = termFreqMap.get(word);
         if (cnt == null)
         {
            termFreqMap.put(word, new Int());
         }
         else
         {
            cnt.x++;
         }
      }
      ts.end();
      ts.close();
   }
622850
def normalize_columns(features):
    
    # Calculate inverse document frequency.
    features = features.tocsc()
    for j in range(features.shape[1]):
        document_frequency = features.getcol(j).data.size
        if document_frequency > 1:
            features.data[features.indptr[j]: features.indptr[j + 1]] =\
                features.data[features.indptr[j]: features.indptr[j + 1]]/np.sqrt(np.log(document_frequency))

    features = features.tocsr()

    return features
772033
function( term ) {
    
    var document_count
    var appearances = 1
    
    for( key in this.documents ) {
      this.documents[ key ][ term ]
        ? appearances++ : 0
    }
    
    document_count = Object.keys( this.documents )
    document_count = document_count.length + 1
    
    return Math.log( document_count / appearances )
    
  }
938550
private void calcNumWords() {
        if (numWordsCalculated) {
            return;
        }

        int words = 0;

        // The number of words reachable from this node is considered to be the number of words
        // reachable from it's children, and +1 if the node itself is a word.
        if (value.isPresent()) {
            words++;
        }

        for (TrieNode<T> child : children.values()) {
            child.calcNumWords();
            words += child.numWords;
        }

        numWords = words;
        numWordsCalculated = true;
    }
620294
public function calculateDensities()
		{
			if ($this->numWordsInWrappedLines == 0)
			{
				$this->numWordsInWrappedLines = $this->numWords;
				$this->numWrappedLines = 1;
			}
			$this->textDensity = $this->numWordsInWrappedLines / $this->numWrappedLines;
			$this->linkDensity = $this->numWords == 0 ? 0 : $this->numWordsInAnchorText / $this->numWords;
			
			// Set full text words only if this block is past the threshold
			if($this->textDensity >= self::TEXT_DENSITY_THRESHOLD)
			{
				$this->numFullTextWords = $this->numWords;
			}
		}
1241378
public int getTermFrequency(String term)
    {
        TermFrequency termFrequency = trieSingle.get(term);
        if (termFrequency == null) return 0;
        return termFrequency.getValue();
    }
844398
def get_info_content(go_id, termcounts):
    '''
        
    '''
    # Get the observed frequency of the GO term
    freq = termcounts.get_term_freq(go_id)

    # Calculate the information content (i.e., -log("freq of GO term")
    return -1.0 * math.log(freq) if freq else 0
280028
public static int getFrequency(String word)
    {
        CoreDictionary.Attribute attribute = getAttribute(word);
        if (attribute == null) return 0;
        return attribute.totalFrequency;
    }
1210117
public static function __calcFreq()
    {
        foreach (self::$original_freq as $key => $value) {
            self::$FREQ[$key] = log($value / self::$total);
        }
        self::$min_freq = min(self::$FREQ);
    }
1212210
def get_ranks(self):
		'''
		

		'''
		if self._use_non_text_features:
			return self._term_doc_matrix.get_metadata_freq_df()
		else:
			return self._term_doc_matrix.get_term_freq_df()
247524
function(err, results) {
        if (err) {
            callback(err);
            return;
        }
        
        var keyFrequency = parseInt(results[0]);
        var docFrequency = parseInt(results[1]);
        var keyDocFrequency = results[2];

        // ensure that key frequency is relative to the given document (when appropriate)
        if (id && results.length == 4) {
            keyFrequency = parseInt(results[3]);
        }
        var tf = 1 + (Math.log(keyFrequency) / Math.LN10);
        var idf = Math.log(docFrequency / keyDocFrequency) / Math.LN10;

        var result = {};
        result.key = key;
        result.rawtf = keyFrequency;
        result.df = keyDocFrequency;
        result.n = docFrequency;
        result.idf = idf;
        result.tf = tf;
        result.tfidf = tf * idf;
        callback(null, result);
    }