Accumulate statistics on how often each token value occurs
436994
public void train(StringWrapperIterator i) 
	{
		Set seenTokens = new HashSet();
		while (i.hasNext()) {
			BagOfTokens bag = asBagOfTokens(i.nextStringWrapper());
			seenTokens.clear();
			for (Iterator j=bag.tokenIterator(); j.hasNext(); ) {
				totalTokenCount++;
				Token tokj = (Token)j.next();
				if (!seenTokens.contains(tokj)) {
					seenTokens.add(tokj);
					// increment documentFrequency counts
					Integer df = (Integer)documentFrequency.get(tokj);
					if (df==null) documentFrequency.put(tokj,ONE); 
					else if (df==ONE) documentFrequency.put(tokj,TWO);
					else if (df==TWO) documentFrequency.put(tokj,THREE);
					else documentFrequency.put(tokj, new Integer(df.intValue()+1));
				}
			}
			collectionSize++;
		}
	}
1261521
def next_tokens_in_sequence(observed, current):
    
    idx = 0
    for word in current:
        if observed[idx:].count(word) != 0:
            found_pos = observed.index(word, idx)
            idx = max(idx + 1, found_pos)
        # otherwise, don't increment idx
    if idx < len(observed):
        return observed[idx:]
    else:
        return []
740739
def calculate_probabilities(tokens)
      # P(class|words) = P(w1,...,wn|class) * P(class) / P(w1,...,wn)
      #                = argmax P(w1,...,wn|class) * P(class)
      #
      # P(wi|class) = (count(wi, class) + k)/(count(w,class) + kV)
      prob_numerator = {}
      v_size = vocab.size

      cat_prob = Math.log(1 / data.categories.count.to_f)
      total_example_count = data.total_examples.to_f

      data.each do |category|
        unless assume_uniform
          cat_prob = Math.log(data.example_count(category) / total_example_count)
        end

        log_probs = 0
        denominator = (data.token_count(category) + @k * v_size).to_f
        tokens.each do |token|
          numerator = data.count_of_token_in_category(category, token) + @k
          log_probs += Math.log( numerator / denominator )
        end
        prob_numerator[category] = log_probs + cat_prob
      end
      normalize(prob_numerator)
    end
1178225
def _aggregate(self, source, aggregators, data, result):
    """"""
    if data is None:
      return

    if hasattr(aggregators, 'items'):
      # Keep walking the tree.
      for key, value in six.iteritems(aggregators):
        if isinstance(key, tuple):
          key, regex = key
          for dataKey, dataValue in six.iteritems(data):
            if regex.match(dataKey):
              result.setdefault(key, {})
              self._aggregate(source, value, dataValue, result[key])
        else:
          if key == '*':
            for dataKey, dataValue in six.iteritems(data):
              result.setdefault(dataKey, {})
              self._aggregate(source, value, dataValue, result[dataKey])
          elif key in data:
            result.setdefault(key, {})
            self._aggregate(source, value, data[key], result[key])

    else:
      # We found a leaf.
      for aggregator in aggregators:
        if aggregator.name not in result:
          result[aggregator.name] = aggregator.clone()
        result[aggregator.name].addValue(source, data)
962213
def get_token_and_data(self, data):
        '''
        
        '''
        token = ''
        for c in data:
            if c != ')':
                token = token + c
            else:
                break;

        return token, data.lstrip(token + '):')
436293
def collect_tokens(cls, parseresult, mode):
        
        inner = '(%s)' if mode=='parens' else '[%s]'
        if parseresult is None: return []
        tokens = []
        for token in parseresult.asList():
            # If value is a tuple, the token will be a list
            if isinstance(token, list):
                token = cls.recurse_token(token, inner)
                tokens[-1] = tokens[-1] + token
            else:
                if token.strip() == ',': continue
                tokens.append(cls._strip_commas(token))
        return tokens
450293
def daily_occurrences(self, dt=None):
        '''
        
        '''
        return Occurrence.objects.daily_occurrences(dt=dt, event=self)
935867
function findTokenIndex(tokens, offset) {
  var min = 0;
  var max = tokens.length - 1;

  while (min <= max) {
    var ptr = (min + max) / 2 | 0;
    var token = tokens[ptr];
    if (token.end <= offset) {
      min = ptr + 1;
    } else if (token.start > offset) {
      max = ptr - 1;
    } else {
      return ptr;
    }
  }

  return ptr;
}
1052945
def count_tf(tokens_stream):
    
    tf = defaultdict(int)
    for tokens in tokens_stream:
        for token in tokens:
            tf[token] += 1
    return tf
218307
def prob(self, pw):
        

        tokens = self.pcfgtokensofw(pw)
        S, tokens = tokens[0], tokens[1:]
        l = len(tokens)
        assert l % 2 == 0, "Expecting even number of tokens!. got {}".format(tokens)

        p = float(self._T.get(S, 0.0)) / sum(v for k, v in self._T.items('__S__'))
        for i, t in enumerate(tokens):
            f = self._T.get(t, 0.0)
            if f == 0:
                return 0.0
            if i < l / 2:
                p /= f
            else:
                p *= f
                # print pw, p, t, self._T.get(t)
        return p
1224840
def count(self, stats, value, sample_rate=1):
        
        self.update_stats(stats, value, self.SC_COUNT, sample_rate)
214022
def rsl_push_many_readings(self, value, count, stream_id):
        """"""

        #FIXME: Fix this with timestamp from clock manager task

        for i in range(1, count+1):
            err = self.sensor_log.push(stream_id, 0, value)
            if err != Error.NO_ERROR:
                return [err, i]

        return [Error.NO_ERROR, count]
951970
def ngram(n, iter_tokens):
    
    z = len(iter_tokens)
    return (iter_tokens[i:i+n] for i in range(z-n+1))
878280
def count_events():
    
    # wait loop
    while 1:
        # sleep and let the queue build up
        gevent.sleep(FLUSH_INTERVAL)

        # init the data points containers
        events = Counter()
        additional = {}

        # flush the queue
        while 1:
            try:
                site, content_id, event, path = EVENTS_QUEUE.get_nowait()
                events[(site, content_id)] += 1
                if event and path:
                    additional[(site, content_id)] = (event, path)
            except queue.Empty:
                break
            except Exception as e:
                LOGGER.exception(e)
                break

        # after tabulating, spawn a new thread to send the data to influxdb
        if len(events):
            gevent.spawn(send_point_data, events, additional)
            gevent.spawn(send_trending_data, events)
989291
def _accumulate(self, old_accum, next_val):
        
        return old_accum * (1 - self.alpha) + next_val * self.alpha
762156
def count(self, token_list):
        '''
        
        '''
        token_dict = {}
        for token in token_list:
            if token in token_dict:
                token_dict[token] += 1
            else:
                token_dict[token] = 1
        return token_dict
374409
def numval(token):
    """"""
    if token.type == 'INTEGER':
        return int(token.value)
    elif token.type == 'FLOAT':
        return float(token.value)
    else:
        return token.value