Returns the number of tokens in the article.
339320
private int getTokenCount(String article) {
        Pattern notWhiteSpace = Pattern.compile("\\S+");
        Matcher matcher = notWhiteSpace.matcher(article);
        int tokens = 0;
        while (matcher.find())
            tokens++;
        return tokens;
    }
534736
private static List<Token> getInlineDocumentation(Token token, int line) {
    List<Token> comments = new ArrayList<>();

    for (Trivia trivia : token.getTrivia()) {
      if (trivia.isComment()) {
        Token triviaToken = trivia.getToken();
        if ((triviaToken != null)
          && (triviaToken.getLine() == line)
          && (isDoxygenInlineComment(triviaToken.getValue()))) {
          comments.add(triviaToken);
          if (LOG.isTraceEnabled()) {
            LOG.trace("Inline doc: " + triviaToken.getValue());
          }
        }
      }
    }
    return comments;
  }
104251
def _get_num_tokens_from_first_line(line: str) -> Optional[int]:
        
        fields = line.split(' ')
        if 1 <= len(fields) <= 2:
            try:
                int_fields = [int(x) for x in fields]
            except ValueError:
                return None
            else:
                num_tokens = max(int_fields)
                logger.info('Recognized a header line in the embedding file with number of tokens: %d',
                            num_tokens)
                return num_tokens
        return None
295348
def number_of_extents(self):
    """"""
    if not self._is_parsed:
      self._Parse()
      self._is_parsed = True

    return len(self._extents)
28243
def articles(self):
		'''  '''

		result = [None, None]
		element = self._first('NN')
		if element:
			element = element.split('\r\n')[0]
			if ' | ' in element:
				# This means there is a plural
				singular, plural = element.split(' | ')
				singular, plural = singular.strip(), plural.strip()
			else:
				# This means there is no plural
				singular, plural = element.strip(), ''
				result[1] = ''
			if singular:
				result[0] = singular.split(' ')[0].split('/')
			if plural:
				result[1] = plural.split(' ')[0].split('/')
		return result
459096
public function getTokenCountUnanswered()
    {
        $tokenSelect = $this->tracker->getTokenSelect(new \Zend_Db_Expr('COUNT(*)'));
        $tokenSelect
                ->andReceptionCodes(array())
                ->andSurveys(array())
                ->andRounds(array())
                ->forRespondent($this->getRespondentId())
                ->forGroupId($this->getSurvey()->getGroupId())
                ->onlySucces()
                ->onlyValid()
                ->forWhere('gsu_active = 1')
                ->forWhere('gro_active = 1 OR gro_active IS NULL')
                ->withoutToken($this->_tokenId);

        $this->_addRelation($tokenSelect);

        return $tokenSelect->fetchOne();
    }
820095
def tag_counts(self):
                            
        qry = self.ses.query(SymbolTag.tag, func.count(SymbolTag.tag))
        qry = qry.group_by(SymbolTag.tag)
        qry = qry.order_by(SymbolTag.tag)
        tags = list(qry.all())
        return tags
1053497
def _parse_next_token(self):
        """"""
        while self._position < self.limit:
            token = self._next_pattern()
            if token:
                return token

        return None
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens
1198380
def _parse_tokens(tokens):
    

    index = 0
    parsed_tokens = []

    num_tokens = len(tokens)
    while index < num_tokens:
        tok = Token(*tokens[index])

        assert tok.token_type != token.INDENT
        if tok.token_type == tokenize.NEWLINE:
            # There's only one newline and it's at the end.
            break

        if tok.token_string in '([{':
            (container, index) = _parse_container(tokens, index)
            if not container:
                return None
            parsed_tokens.append(container)
        else:
            parsed_tokens.append(Atom(tok))

        index += 1

    return parsed_tokens
433648
def detailed_tokens(tokenizer, text):
    """"""
    node = tokenizer.parseToNode(text)
    node = node.next  # first node is beginning of sentence and empty, skip it
    words = []
    while node.posid != 0:
        surface = node.surface
        base = surface  # a default value. Updated if available later.
        parts = node.feature.split(",")
        pos = ",".join(parts[0:4])
        if len(parts) > 7:
            # this information is only available for words in the tokenizer
            # dictionary
            base = parts[7]
        words.append(ShortUnitWord(surface, base, pos))
        node = node.next
    return words
103168
def count(self):
        ""
        if getattr(self, '_count', None) is None:
            self._count = getattr(self.node, 'count', 0)
        return self._count
860492
protected int countToken(String token, String target) {
        int tokenIndex = 0;
        int count = 0;
        while (tokenIndex != -1) {
            tokenIndex = target.indexOf(token, tokenIndex);
            if (tokenIndex > -1) {
                tokenIndex++;
                count++;
            }
        }
        return count;
    }
189836
def tokentype(self, s):
        """"""
        a = s[0] if len(s)>0 else ""
        b = s[1] if len(s)>1 else ""

        if a.isdigit() or (a in ["+","-"] and b.isdigit()):
            return self.parse_number(s)
        elif a == '"': return self.parse_string(s)
        elif a == ':': return self.parse_colon(s)
        elif a == ';': return self.parse_semicolon(s)
        else:
            return self.parse_word(s)
685695
def follower_num(self):
        
        href = re_collection_url_split.match(self.url).group(1)
        return int(self.soup.find('a', href=href + 'followers').text)
611227
def size(self):
        ''''''
        count = 0
        cursor = self._db_coll.find()
        for item in cursor:
            if item['origin'] == '@_abbreviations':
                continue
            count += len(item['rels'])
        return count
886679
def word_counts(self):
    
    counts = defaultdict(int)
    for word in self.words:
        counts[word] += 1
    return counts