means that there will be another token with or without more input.
433507
private boolean hasTokenInBuffer() {
        matchValid = false;
        matcher.usePattern(delimPattern);
        matcher.region(position, buf.limit());

        // Skip delims first
        if (matcher.lookingAt())
            position = matcher.end();

        // If we are sitting at the end, no more tokens in buffer
        if (position == buf.limit())
            return false;

        return true;
    }
436716
private final void prevToken()
  {

    if (m_queueMark > 0)
    {
      m_queueMark--;

      m_token = (String) m_ops.m_tokenQueue.elementAt(m_queueMark);
      m_tokenChar = m_token.charAt(0);
    }
    else
    {
      m_token = null;
      m_tokenChar = 0;
    }
  }
429449
function (p) {
                var i, t = p;
                if (nexttoken.id) {
                    if (!t) {
                        t = '';
                        if (nexttoken.id.substr(0, 1) === '<') {
                            lookahead.push(nexttoken);
                            return true;
                        }
                    } else if (nexttoken.id.indexOf(t) >= 0) {
                        return true;
                    }
                }
                token = nexttoken;
                nexttoken = syntax['(end)'];
                for (;;) {
                    i = s.indexOf(t || '<');
                    if (i >= 0) {
                        character += i + t.length;
                        s = s.substr(i + t.length);
                        return true;
                    }
                    if (!nextLine()) {
                        break;
                    }
                }
                return false;
            }
1320449
function (token) {
            if (token.prev.type !== 'WhiteSpace') {
                insertBefore(token, whiteSpaceFactory());
            }
            if (token.next.type !== 'WhiteSpace' && token.next.type !== 'LineBreak') {
                insertAfter(token, whiteSpaceFactory());
            }
        }
239507
function handleEscaping(tokens, token, index) {
	if (index === 0) {
		return [token];
	}

	const previousToken = tokens[index - 1];

	if (!previousToken.endsWith('\\')) {
		return [...tokens, token];
	}

	return [...tokens.slice(0, index - 1), `${previousToken.slice(0, -1)} ${token}`];
}
921889
function isOnSameLineWithNextToken(node) {
            const prevToken = sourceCode.getLastToken(node, 1);
            const nextToken = sourceCode.getTokenAfter(node);

            return !!nextToken && astUtils.isTokenOnSameLine(prevToken, nextToken);
        }
568285
boolean addNoParaToken(Token noParaToken) {
		tokens.add(noParaToken);
		if (CharTable.isBlank(peek())) {
			next();	// 无参指令之后紧随的一个空白字符仅为分隔符，不参与后续扫描
		}
		
		if (lookForwardLineFeedAndEof() && deletePreviousTextTokenBlankTails()) {
			prepareNextScan(peek() != EOF ? 1 : 0);
		} else {
			prepareNextScan(0);
		}
		previousTextToken = null;
		return true;
	}
589030
def _expect_empty(self):
        

        item = self._lexer.get_token()
        if item:
            line_no, token = item
            raise ParseError(u"Unexpected token '{0}' on line {1}"
                             .format(common.from_utf8(token.strip()), line_no))
461991
function nextToken() {
            if (curOffset >= length) {
                return false;
            }
            if (stream) {
                // Set the start of the next token to the current stream position.
                stream.start = stream.pos;
            }
            while (!stream || stream.eol()) {
                if (!nextLine()) {
                    return false;
                }
            }
            style = mode.token(stream, state);
            token = stream.current();
            curOffset = lineStart + stream.pos;
            return true;
        }
846841
private void advance() {
	String s = null;
	while (tokenizer.hasNext()) {
	    String nextToken = tokenizer.next();
	    // stop if the word should be returned
	    if (filter.accept(nextToken)) {
		s = nextToken;
		break;
	    }
	}
	next = s;
    }
1255203
private static String nextToken(String[] tokens, int x, String context)
            throws ParseException {
        if (x < tokens.length) {
            return tokens[x];
        }
        throw new ParseException("missing token at end of '" + context + "'", -1);
    }
1078896
protected final Token consumeToken() throws ScanException, ParseException {
		Token result = token;
		if (lookahead.isEmpty()) {
			token = scanner.next();
			position = scanner.getPosition();
		} else {
			LookaheadToken next = lookahead.remove(0);
			token = next.token;
			position = next.position;
		}
		return result;
	}
970875
private Token nextToken() {
        skipSpaces();
        if (hasNextChar()) {
            char ch = nextChar();
            if (Character.isJavaIdentifierStart(ch)) {
                StringBuffer buf = new StringBuffer();
                buf.append(ch);
                while ((ch = peekChar()) != -1
                        && Character.isJavaIdentifierPart(ch)) {
                    buf.append(ch);
                    nextChar();
                }
                return new Id(buf.toString());
            }

            if (ch == '\'' || ch == '"') {
                return parseQuotedChars(ch);
            } else {
                // For now...
                return new Char(ch);
            }
        }
        return null;
    }
86588
private boolean tokenTextsDoNotMatch(Token token1, Token token2) {
        return !token1.getText().equals(token2.getText());
    }
1161883
def NextToken(self):
    
    # If we don't have enough data - feed ourselves: We assume
    # that we must have at least one sector in our buffer.
    if len(self.buffer) < 512:
      if self.Feed() == 0 and not self.buffer:
        return None

    return Lexer.NextToken(self)
1051897
public Token next() throws ScanException {
		if (token != null) {
			position += token.getSize();
		}
	
		int length = input.length();
				
		if (isEval()) {
			while (position < length && Character.isWhitespace(input.charAt(position))) {
				position++;
			}
		}

		if (position == length) {
			return token = fixed(Symbol.EOF);
		}

		return token = nextToken();
	}
67481
public function getNextToken() {
		if ($this->index < (count($this->tokens) - 1)) {

			// Increment the index
			$this->index ++;

			// Return the new token
			return $this->tokens[$this->index];
		} else {
			return false;
		}
	}