Reads the token after the given token.

@param Token $prev
@return Token
@throws SyntaxErrorException
332124
protected function readToken(Token $prev): Token
    {
        $this->position = $prev->getEnd();

        $this->skipWhitespace();

        $line   = $this->line;
        $column = (1 + $this->position) - $this->lineStart;

        if ($this->position >= $this->bodyLength) {
            return $this->createEndOfFileToken($line, $column, $prev);
        }

        $code = $this->readCharCode($this->position);

        // Punctuation: [!$&:=@|()\[\]{}]{1}
        if (33 === $code || 36 === $code || 38 === $code || 58 === $code || 61 === $code || 64 === $code || 124 === $code ||
            40 === $code || 41 === $code || 91 === $code || 93 === $code || 123 === $code || 125 === $code) {
            return $this->lexPunctuation($code, $line, $column, $prev);
        }

        // Comment: #[\u0009\u0020-\uFFFF]*
        if (35 === $code) {
            return $this->lexComment($line, $column, $prev);
        }

        // Int:   -?(0|[1-9][0-9]*)
        // Float: -?(0|[1-9][0-9]*)(\.[0-9]+)?((E|e)(+|-)?[0-9]+)?
        if (45 === $code || isNumber($code)) {
            return $this->lexNumber($code, $line, $column, $prev);
        }

        // Name: [_A-Za-z][_0-9A-Za-z]*
        if (isAlphaNumeric($code)) {
            return $this->lexName($line, $column, $prev);
        }

        // Spread: ...
        if ($this->bodyLength >= 3 && $this->isSpread($code)) {
            return $this->lexSpread($line, $column, $prev);
        }

        // String: "([^"\\\u000A\u000D]|(\\(u[0-9a-fA-F]{4}|["\\/bfnrt])))*"
        if ($this->isString($code)) {
            return $this->lexString($line, $column, $prev);
        }

        // Block String: """("?"?(\\"""|\\(?!=""")|[^"\\]))*"""
        if ($this->bodyLength >= 3 && $this->isTripleQuote($code)) {
            return $this->lexBlockString($line, $column, $prev);
        }

        throw $this->createSyntaxErrorException();
    }
692596
protected function _getNextToken()
    {
        $this->token      = self::EOF;
        $this->tokenValue = null;
        $this->_eatWhitespace();

        if ($this->offset >= $this->sourceLength) {
            return(self::EOF);
        }

        $str       = $this->source;
        $strLength = $this->sourceLength;
        $i         = $this->offset;
        $start     = $i;

        switch ($str{$i}) {
            case '{':
                $this->token = self::LBRACE;
                break;
            case '}':
                $this->token = self::RBRACE;
                break;
            case '[':
                $this->token = self::LBRACKET;
                break;
            case ']':
                $this->token = self::RBRACKET;
                break;
            case ',':
                $this->token = self::COMMA;
                break;
            case ':':
                $this->token = self::COLON;
                break;
            case '"':
                $result = '';
                do {
                    $i++;
                    if ($i >= $strLength) {
                        break;
                    }

                    $chr = $str{$i};

                    if ($chr == '\\') {
                        $i++;
                        if ($i >= $strLength) {
                            break;
                        }
                        $chr = $str{$i};
                        switch ($chr) {
                            case '"':
                                $result .= '"';
                                break;
                            case '\\':
                                $result .= '\\';
                                break;
                            case '/':
                                $result .= '/';
                                break;
                            case 'b':
                                $result .= "\x08";
                                break;
                            case 'f':
                                $result .= "\x0c";
                                break;
                            case 'n':
                                $result .= "\x0a";
                                break;
                            case 'r':
                                $result .= "\x0d";
                                break;
                            case 't':
                                $result .= "\x09";
                                break;
                            case '\'':
                                $result .= '\'';
                                break;
                            default:
                                throw new RuntimeException("Illegal escape sequence '{$chr}'");
                        }
                    } elseif ($chr == '"') {
                        break;
                    } else {
                        $result .= $chr;
                    }
                } while ($i < $strLength);

                $this->token = self::DATUM;
                //$this->tokenValue = substr($str, $start + 1, $i - $start - 1);
                $this->tokenValue = $result;
                break;
            case 't':
                if (($i+ 3) < $strLength && substr($str, $start, 4) == "true") {
                    $this->token = self::DATUM;
                }
                $this->tokenValue = true;
                $i += 3;
                break;
            case 'f':
                if (($i+ 4) < $strLength && substr($str, $start, 5) == "false") {
                    $this->token = self::DATUM;
                }
                $this->tokenValue = false;
                $i += 4;
                break;
            case 'n':
                if (($i+ 3) < $strLength && substr($str, $start, 4) == "null") {
                    $this->token = self::DATUM;
                }
                $this->tokenValue = null;
                $i += 3;
                break;
        }

        if ($this->token != self::EOF) {
            $this->offset = $i + 1; // Consume the last token character
            return($this->token);
        }

        $chr = $str{$i};
        if ($chr == '-' || $chr == '.' || ($chr >= '0' && $chr <= '9')) {
            if (preg_match('/-?([0-9])*(\.[0-9]*)?((e|E)((-|\+)?)[0-9]+)?/s', $str, $matches, PREG_OFFSET_CAPTURE, $start) && $matches[0][1] == $start) {
                $datum = $matches[0][0];

                if (is_numeric($datum)) {
                    if (preg_match('/^0\d+$/', $datum)) {
                        throw new RuntimeException("Octal notation not supported by JSON (value: {$datum})");
                    } else {
                        $val  = intval($datum);
                        $fVal = floatval($datum);
                        $this->tokenValue = ($val == $fVal ? $val : $fVal);
                    }
                } else {
                    throw new RuntimeException("Illegal number format: {$datum}");
                }

                $this->token = self::DATUM;
                $this->offset = $start + strlen($datum);
            }
        } else {
            throw new RuntimeException('Illegal Token');
        }

        return $this->token;
    }
584498
public function _getPrev(&$tokens, $key, $whatToGet)
    {
        $key--;
        if (!is_array($whatToGet)) $whatToGet = array($whatToGet);
        while (!is_array($tokens[$key]) || !in_array($tokens[$key][0], $whatToGet)) {
            $key--;
            if (!isset($tokens[$key])) return FALSE;
        }

        return $tokens[$key][1];
    }
949683
public function checkPreviousToken($id, $text = false) {
		$token = $this->peekPrvsToken();
		return $this->checkToken($token, $id, $text);
	}
1010231
public function expect($type, $value = null, $message = null)
    {
        $token = $this->current;
        if (!$token->test($type, $value)) {
            throw new SyntaxError(sprintf('%sUnexpected token "%s" of value "%s" ("%s" expected%s)', $message ? $message.'. ' : '', $token->type, $token->value, $type, $value ? sprintf(' with value "%s"', $value) : ''), $token->cursor);
        }
        $this->next();
    }
595751
public function expect(string $field, string $pre = ''): ExpectTool
    {
        return new ExpectTool(
            $this,
            $this->cnn,
            $this->condPart->expect($field, $pre)
        );
    }
745120
private Object getNext(String txt, String originalText) {
    if (invertible) {
      String str = prevWordAfter.toString();
      prevWordAfter.setLength(0);
      CoreLabel word = (CoreLabel) tokenFactory.makeToken(txt, yychar, yylength());
      word.set(OriginalTextAnnotation.class, originalText);
      word.set(BeforeAnnotation.class, str);
      prevWord.set(AfterAnnotation.class, str);
      prevWord = word;
      return word;
    } else {
      return tokenFactory.makeToken(txt, yychar, yylength());
   }
  }
273804
private function reRegister($prev)
    {
        if ($prev !== $this->thrownErrors | $this->loggedErrors) {
            $handler = set_error_handler('var_dump');
            $handler = \is_array($handler) ? $handler[0] : null;
            restore_error_handler();
            if ($handler === $this) {
                restore_error_handler();
                if ($this->isRoot) {
                    set_error_handler([$this, 'handleError'], $this->thrownErrors | $this->loggedErrors);
                } else {
                    set_error_handler([$this, 'handleError']);
                }
            }
        }
    }
800445
protected function createEndOfFileToken(int $line, int $column, Token $prev): Token
    {
        return new Token(TokenKindEnum::EOF, $this->bodyLength, $this->bodyLength, $line, $column, $prev);
    }
937568
public function parse()
    {
        /**
         * Last transaction.
         *
         * @var TransactionStatement
         */
        $lastTransaction = null;

        /**
         * Last parsed statement.
         *
         * @var Statement
         */
        $lastStatement = null;

        /**
         * Union's type or false for no union.
         *
         * @var bool|string
         */
        $unionType = false;

        /**
         * The index of the last token from the last statement.
         *
         * @var int
         */
        $prevLastIdx = -1;

        /**
         * The list of tokens.
         *
         * @var TokensList
         */
        $list = &$this->list;

        for (; $list->idx < $list->count; ++$list->idx) {
            /**
             * Token parsed at this moment.
             *
             * @var Token
             */
            $token = $list->tokens[$list->idx];

            // `DELIMITER` is not an actual statement and it requires
            // special handling.
            if (($token->type === Token::TYPE_NONE)
                && (strtoupper($token->token) === 'DELIMITER')
            ) {
                // Skipping to the end of this statement.
                $list->getNextOfType(Token::TYPE_DELIMITER);
                $prevLastIdx = $list->idx;
                continue;
            }

            // Counting the brackets around statements.
            if ($token->value === '(') {
                ++$this->brackets;
                continue;
            }

            // Statements can start with keywords only.
            // Comments, whitespaces, etc. are ignored.
            if ($token->type !== Token::TYPE_KEYWORD) {
                if (($token->type !== Token::TYPE_COMMENT)
                    && ($token->type !== Token::TYPE_WHITESPACE)
                    && ($token->type !== Token::TYPE_OPERATOR) // `(` and `)`
                    && ($token->type !== Token::TYPE_DELIMITER)
                ) {
                    $this->error(
                        'Unexpected beginning of statement.',
                        $token
                    );
                }
                continue;
            }

            if (($token->keyword === 'UNION') ||
                    ($token->keyword === 'UNION ALL') ||
                    ($token->keyword === 'UNION DISTINCT') ||
                    ($token->keyword === 'EXCEPT') ||
                    ($token->keyword === 'INTERSECT')
            ) {
                $unionType = $token->keyword;
                continue;
            }

            // Checking if it is a known statement that can be parsed.
            if (empty(static::$STATEMENT_PARSERS[$token->keyword])) {
                if (! isset(static::$STATEMENT_PARSERS[$token->keyword])) {
                    // A statement is considered recognized if the parser
                    // is aware that it is a statement, but it does not have
                    // a parser for it yet.
                    $this->error(
                        'Unrecognized statement type.',
                        $token
                    );
                }
                // Skipping to the end of this statement.
                $list->getNextOfType(Token::TYPE_DELIMITER);
                $prevLastIdx = $list->idx;
                continue;
            }

            /**
             * The name of the class that is used for parsing.
             *
             * @var string
             */
            $class = static::$STATEMENT_PARSERS[$token->keyword];

            /**
             * Processed statement.
             *
             * @var Statement
             */
            $statement = new $class($this, $this->list);

            // The first token that is a part of this token is the next token
            // unprocessed by the previous statement.
            // There might be brackets around statements and this shouldn't
            // affect the parser
            $statement->first = $prevLastIdx + 1;

            // Storing the index of the last token parsed and updating the old
            // index.
            $statement->last = $list->idx;
            $prevLastIdx = $list->idx;

            // Handles unions.
            if (! empty($unionType)
                && ($lastStatement instanceof SelectStatement)
                && ($statement instanceof SelectStatement)
            ) {
                /*
                 * This SELECT statement.
                 *
                 * @var SelectStatement $statement
                 */

                /*
                 * Last SELECT statement.
                 *
                 * @var SelectStatement $lastStatement
                 */
                $lastStatement->union[] = [
                    $unionType,
                    $statement,
                ];

                // if there are no no delimiting brackets, the `ORDER` and
                // `LIMIT` keywords actually belong to the first statement.
                $lastStatement->order = $statement->order;
                $lastStatement->limit = $statement->limit;
                $statement->order = [];
                $statement->limit = null;

                // The statement actually ends where the last statement in
                // union ends.
                $lastStatement->last = $statement->last;

                $unionType = false;

                // Validate clause order
                $statement->validateClauseOrder($this, $list);
                continue;
            }

            // Handles transactions.
            if ($statement instanceof TransactionStatement) {
                /*
                 * @var TransactionStatement
                 */
                if ($statement->type === TransactionStatement::TYPE_BEGIN) {
                    $lastTransaction = $statement;
                    $this->statements[] = $statement;
                } elseif ($statement->type === TransactionStatement::TYPE_END) {
                    if ($lastTransaction === null) {
                        // Even though an error occurred, the query is being
                        // saved.
                        $this->statements[] = $statement;
                        $this->error(
                            'No transaction was previously started.',
                            $token
                        );
                    } else {
                        $lastTransaction->end = $statement;
                    }
                    $lastTransaction = null;
                }

                // Validate clause order
                $statement->validateClauseOrder($this, $list);
                continue;
            }

            // Validate clause order
            $statement->validateClauseOrder($this, $list);

            // Finally, storing the statement.
            if ($lastTransaction !== null) {
                $lastTransaction->statements[] = $statement;
            } else {
                $this->statements[] = $statement;
            }
            $lastStatement = $statement;
        }
    }
187558
protected function lexPunctuation(int $code, int $line, int $column, Token $prev): ?Token
    {
        if (!isset(self::$codeTokenKindMap[$code])) {
            throw $this->createSyntaxErrorException();
        }

        return new Token(self::$codeTokenKindMap[$code], $this->position, $this->position + 1, $line, $column, $prev);
    }
850615
protected function readToken()
    {
        $inputStream = $this->inputStream;

        $this->stepOverWhitespace();

        $char = $inputStream->readCurrent();

        if ($char === null) {
            return null;
        }

        if ($this->isLetter($char)) {
            $value = $this->readWord();
            $type = Token::TYPE_WORD;
        } elseif ($this->isDigit($char) or $this->isPeriod($char)) {
            $value = $this->readNumber();
            $type = Token::TYPE_NUMBER;
        } else {
            $value = $this->readChar();
            $type = Token::TYPE_CHARACTER;
        }

        $token = new Token($value, $type, $this->inputStream->getPosition());

        return $token;
    }
127412
protected function lexString(int $line, int $column, Token $prev): Token
    {
        $start      = $this->position;
        $chunkStart = ++$this->position; // skip the quote
        $value      = '';

        while ($this->position < $this->bodyLength) {
            $code = $this->readCharCode($this->position);

            if (isLineTerminator($code)) {
                break;
            }

            // Closing Quote (")
            if (34 === $code) {
                $value .= sliceString($this->body, $chunkStart, $this->position);
                return new Token(TokenKindEnum::STRING, $start, $this->position + 1, $line, $column, $prev, $value);
            }

            if (isSourceCharacter($code)) {
                throw $this->createSyntaxErrorException(
                    \sprintf('Invalid character within String: %s.', printCharCode($code))
                );
            }

            ++$this->position;

            if (92 === $code) {
                // \
                $value .= sliceString($this->body, $chunkStart, $this->position - 1);

                $code = $this->readCharCode($this->position);

                switch ($code) {
                    case 34: // "
                        $value .= '"';
                        break;
                    case 47: // /
                        $value .= '/';
                        break;
                    case 92: // \
                        $value .= '\\';
                        break;
                    case 98: // b
                        $value .= '\b';
                        break;
                    case 102: // f
                        $value .= '\f';
                        break;
                    case 110: // n
                        $value .= '\n';
                        break;
                    case 114: // r
                        $value .= '\r';
                        break;
                    case 116: // t
                        $value .= '\t';
                        break;
                    case 117: // u
                        $unicodeString = sliceString($this->body, $this->position + 1, $this->position + 5);

                        if (!\preg_match('/[0-9A-Fa-f]{4}/', $unicodeString)) {
                            throw $this->createSyntaxErrorException(
                                \sprintf('Invalid character escape sequence: \\u%s.', $unicodeString)
                            );
                        }

                        $value .= '\\u' . $unicodeString;

                        $this->position += 4;

                        break;
                    default:
                        throw $this->createSyntaxErrorException(
                            \sprintf('Invalid character escape sequence: \\%s.', \chr($code))
                        );
                }

                ++$this->position;

                $chunkStart = $this->position;
            }
        }

        throw $this->createSyntaxErrorException('Unterminated string.');
    }
973459
public function setTitles($prev = "Prev", $next = "Next")
    {
        $this->options['prev_title'] = $prev;
        $this->options['next_title'] = $next;
        return $this;
    }
105017
public function expect($type, $value = null, $message = null)
    {
        $token = $this->tokens[$this->current];

        if (!$token->test($type, $value)) {
            throw new SyntaxErrorException(sprintf('%sUnexpected token "%s" of value "%s" ("%s" expected%s) on line %s', $message ? "$message. " : "", $token, $token->getValue(), Token::getName($type), $value ? sprintf(' with value "%s"', $value) : '', $token->getLine()));
        }

        return $token;
    }
20481
public function NextToken()
	{
		$anchor = $this->_offset;
		$ch = '\0';
		$this->Value = "";
		while ( $this->_offset < strlen( $this->_text ) )
		{
			$ch = $this->_text[ $this->_offset ];
			if ( preg_match( "/\s/", $ch ) && $ch != '\n' && $ch != '\r' )
				$this->_offset++;
			else
				break;
		}
		switch ( $ch )
		{
			case '\0':

				$this->Token = $ch;
				break;

			case '\n':

				$this->_offset++;
				if ($this->_offset < strlen( $this->_text ) - 1 && $this->_text[ $this->_offset ] == '\r' )
					$this->_offset++;
				$this->LineCount++;
				$this->Token = $ch;
				break;

			case '\r':

				$this->_offset++;
				if ( $this->_offset < strlen( $this->_text.Length ) - 1 && $this->_text[ $this->_offset ] == '\n')
					$this->_offset++;
				$this->LineCount++;
				$this->Token = '\n';
				break;

			default:

				if ( is_numeric( $ch ) )
				{
					$sb = array();
					while ( $this->_offset < strlen( $this->_text ) )
					{
						$ch = $this->_text[ $this->_offset ];
						if ( ! is_numeric( $ch ) )
							break;

						$sb[] = $ch;
						$this->_offset++;
					}
					$this->Token = StringTokenizer::TokenInt;
					$this->Value = implode( "", $sb );
				}
				else
				{
					$this->Token = $ch;
					$this->_offset++;
				}
				break;

		}
		return $this->Token;
	}
744953
protected function _getNextToken()
    {
        $this->_token      = self::EOF;
        $this->_tokenValue = null;
        $this->_eatWhitespace();

        if ($this->_offset >= $this->_sourceLength) {
            return(self::EOF);
        }

        $str        = $this->_source;
        $str_length = $this->_sourceLength;
        $i          = $this->_offset;
        $start      = $i;

        switch ($str{$i}) {
        case '{':
            $this->_token = self::LBRACE;
            break;
        case '}':
            $this->_token = self::RBRACE;
            break;
        case '[':
            $this->_token = self::LBRACKET;
            break;
        case ']':
            $this->_token = self::RBRACKET;
            break;
        case ',':
            $this->_token = self::COMMA;
            break;
        case ':':
            $this->_token = self::COLON;
            break;
        case  '"':
            $result = '';
            do {
                $i++;
                if ($i >= $str_length) {
                    break;
                }

                $chr = $str{$i};
                if ($chr == '\\') {
                    $i++;
                    if ($i >= $str_length) {
                        break;
                    }
                    $chr = $str{$i};
                    switch ($chr) {
                    case '"' :
                        $result .= '"';
                        break;
                    case '\\':
                        $result .= '\\';
                        break;
                    case '/' :
                        $result .= '/';
                        break;
                    case 'b' :
                        $result .= chr(8);
                        break;
                    case 'f' :
                        $result .= chr(12);
                        break;
                    case 'n' :
                        $result .= chr(10);
                        break;
                    case 'r' :
                        $result .= chr(13);
                        break;
                    case 't' :
                        $result .= chr(9);
                        break;
                    case '\'' :
                        $result .= '\'';
                        break;
                    default:
                        throw new Zend_Json_Exception(
                            "Illegal escape "
                                .  "sequence '" . $chr . "'"
                        );
                    }
                } elseif ($chr == '"') {
                    break;
                } else {
                    $result .= $chr;
                }
            } while ($i < $str_length);

            $this->_token = self::DATUM;
            //$this->_tokenValue = substr($str, $start + 1, $i - $start - 1);
            $this->_tokenValue = $result;
            break;
        case 't':
            if (($i+ 3) < $str_length && substr($str, $start, 4) == "true") {
                $this->_token = self::DATUM;
            }
            $this->_tokenValue = true;
            $i += 3;
            break;
        case 'f':
            if (($i+ 4) < $str_length && substr($str, $start, 5) == "false") {
                $this->_token = self::DATUM;
            }
            $this->_tokenValue = false;
            $i += 4;
            break;
        case 'n':
            if (($i+ 3) < $str_length && substr($str, $start, 4) == "null") {
                $this->_token = self::DATUM;
            }
            $this->_tokenValue = null;
            $i += 3;
            break;
        }

        if ($this->_token != self::EOF) {
            $this->_offset = $i + 1; // Consume the last token character
            return($this->_token);
        }

        $chr = $str{$i};
        if ($chr == '-' || $chr == '.' || ($chr >= '0' && $chr <= '9')) {
            if (preg_match(
                '/-?([0-9])*(\.[0-9]*)?((e|E)((-|\+)?)[0-9]+)?/s',
                $str, $matches, PREG_OFFSET_CAPTURE, $start
            ) && $matches[0][1] == $start
            ) {

                $datum = $matches[0][0];

                if (is_numeric($datum)) {
                    if (preg_match('/^0\d+$/', $datum)) {
                        throw new Zend_Json_Exception("Octal notation not supported by JSON (value: $datum)");
                    } else {
                        $val  = intval($datum);
                        $fVal = floatval($datum);
                        $this->_tokenValue = ($val == $fVal ? $val : $fVal);
                    }
                } else {
                    throw new Zend_Json_Exception("Illegal number format: $datum");
                }

                $this->_token = self::DATUM;
                $this->_offset = $start + strlen($datum);
            }
        } else {
            throw new Zend_Json_Exception('Illegal Token');
        }

        return($this->_token);
    }