Notify the tokenizer that the document will be tokenized by repeated step()
calls. This must be called once only, before the first call to step().
493977
public function beginStepping() {
		$this->state = self::STATE_START;
		$this->preprocess();
		$this->listener->startDocument( $this, null, null );
	}
159958
def notify(self, reading):
        """"""
        # copy list before iterating in case new observers arrive
        for o in list(self._observers):
            o.update(self, reading)
238946
private static boolean expect(StringTokenizer tokenizer, String token) {
        if (!tokenizer.hasMoreTokens()) {
            return false;
        }
        String s = tokenizer.nextToken();
        if (DEBUG) {
            System.out.println("token=" + s);
        }
        return s.equals(token);
    }
758914
def once(self):
        
        def check(): #pylint: disable=missing-docstring
            return super(SinonExpectation, self).calledOnce
        self.valid_list.append(check)
        return self
606319
protected function processToken(): void
    {
        $prevStatement = null;

        foreach ($this->useStatements as $useStatement) {
            if ($prevStatement && ($this->compareUseStatements($prevStatement, $useStatement) > 0)) {
                $exception = new CodeError(
                    static::CODE_INCORRECT_ORDER,
                    'Use statements should be sorted alphabetically. The first wrong one is %s.',
                    $useStatement->getPointer()
                );

                $exception
                    ->setPayload([$useStatement->getFullyQualifiedTypeName()])
                    ->isFixable(true);

                throw $exception;
            }

            $prevStatement = $useStatement;
        }
    }
878415
public void insertAfter(Token before, TokenList list ) {
        Token after = before.next;

        before.next = list.first;
        list.first.previous = before;
        if( after == null ) {
            last = list.last;
        } else {
            after.previous = list.last;
            list.last.next = after;
        }
        size += list.size;
    }
436716
private final void prevToken()
  {

    if (m_queueMark > 0)
    {
      m_queueMark--;

      m_token = (String) m_ops.m_tokenQueue.elementAt(m_queueMark);
      m_tokenChar = m_token.charAt(0);
    }
    else
    {
      m_token = null;
      m_tokenChar = 0;
    }
  }
461991
function nextToken() {
            if (curOffset >= length) {
                return false;
            }
            if (stream) {
                // Set the start of the next token to the current stream position.
                stream.start = stream.pos;
            }
            while (!stream || stream.eol()) {
                if (!nextLine()) {
                    return false;
                }
            }
            style = mode.token(stream, state);
            token = stream.current();
            curOffset = lineStart + stream.pos;
            return true;
        }
340856
def check_unique_tokens(sender, instance, **kwargs):
    
    if isinstance(instance, CallbackToken):
        if CallbackToken.objects.filter(key=instance.key, is_active=True).exists():
            instance.key = generate_numeric_token()
468530
def _phrase_doc_stream(paths, n, tokenizer=word_tokenize):
    
    i = 0
    p = Progress()
    for path in paths:
        with open(path, 'r') as f:
            for line in f:
                i += 1
                p.print_progress(i/n)
                for sent in sent_tokenize(line.lower()):
                    tokens = tokenizer(sent)
                    yield tokens
846841
private void advance() {
	String s = null;
	while (tokenizer.hasNext()) {
	    String nextToken = tokenizer.next();
	    // stop if the word should be returned
	    if (filter.accept(nextToken)) {
		s = nextToken;
		break;
	    }
	}
	next = s;
    }
638460
protected function processToken(): void
    {
        $this->checkLinesBeforeFirstUse($this->uses[0]);
        $this->checkLinesAfterLastUse($this->uses[count($this->uses) - 1]);

        if (count($this->uses) > 1) {
            $this->checkLinesBetweenUses();
        }
    }
618078
private int nextToken(StreamTokenizer tokeniser, Reader in, boolean ignoreEOF) throws IOException, ParserException {
        int token = tokeniser.nextToken();
        if (!ignoreEOF && token == StreamTokenizer.TT_EOF) {
            throw new ParserException("Unexpected end of file", getLineNumber(tokeniser, in));
        }
        return token;
    }
1056923
def process_token(self, tok):
        """"""
        if(tok[0].__str__() in ('Token.Comment.Multiline', 'Token.Comment',
                'Token.Literal.String.Doc')):
            self.comments += tok[1].count('\n')+1
        elif(tok[0].__str__() in ('Token.Comment.Single')):
            self.comments += 1
        elif(self.contains_code and tok[0].__str__().startswith('Token.Text')
                and tok[1].count(u'\n')):
            # start new line
            self.contains_code = False
            self.sloc += 1
        # for c style includes
        elif(tok[0].__str__() == 'Token.Comment.Preproc' and
                tok[1].count(u'\n')):
            # start new line
            self.contains_code = False
            self.sloc += 1
        elif(tok[0][0] in token_types):
            self.contains_code = True
222822
public void process(Iterator<String> text) {
        String nextToken = null, curToken = null;
        // Base case for the next token buffer to ensure we always have two
        // valid tokens present
        if (text.hasNext())
            nextToken = text.next();
        while (text.hasNext()) {
            curToken = nextToken;
            nextToken = text.next();
            // Only process bigrams where the two tokens weren't excluded by the
            // token filter
            if (!(excludeToken(curToken) || excludeToken(nextToken)))
                processBigram(curToken, nextToken);
        }
    }
1211146
void initWithSingleTokenPath(String path) {
        this.path = path;
        this.index = 0;
        this.offset = 0;
        this.nextSplit = -1;
        this.token = path;
    }
165979
private void prepareNext()
    {
        hasNext = false;
        while (iterator.hasNext())
        {
            next = iterator.next();
            if (predicate.test(next))
            {
                hasNext = true;
                break;
            }
        }
    }