@param Tokens $tokens
@param int    $index

@return string
432636
private function detectIndent(Tokens $tokens, $index)
	{
		while (true) {
			$whitespaceIndex = $tokens->getPrevTokenOfKind($index, [[T_WHITESPACE]]);

			if ($whitespaceIndex === null) {
				return '';
			}

			$whitespaceToken = $tokens[$whitespaceIndex];

			if (strpos($whitespaceToken->getContent(), "\n") !== false) {
				break;
			}

			$prevToken = $tokens[$whitespaceIndex - 1];

			if ($prevToken->isGivenKind([T_OPEN_TAG, T_COMMENT]) && substr($prevToken->getContent(), -1) === "\n") {
				break;
			}

			$index = $whitespaceIndex;
		}

		$explodedContent = explode("\n", $whitespaceToken->getContent());

		return end($explodedContent);
	}
1247371
protected function _findTagIndex(array $tokens, $from, $to, $tagType) {
		for ($i = $from + 1; $i < $to; $i++) {
			if ($tokens[$i]['code'] === $tagType) {
				return $i;
			}
		}

		return null;
	}
1282132
public static function create(array $tokens) {
        HashMap::assert($tokens, 'tokens');

        if (empty($tokens['WHERE'])) return '';

        $idAlias = self::alias($tokens);

        return array_reduce($tokens['WHERE'], function($carry, $token) use ($tokens, $idAlias) {
            if ($carry !== null) return (string)Value::create($carry);
            if ($token['expr_type'] === 'colref' && $token['base_expr'] === $idAlias) return $tokens['WHERE'][$carry+2]['base_expr'];
            if (!isset($tokens[$carry+1])) return '';
        }, null);
    }
608547
protected function filterTokens(array $tokens)
    {
        $filteredTokens = array_filter($tokens, function (Token $t) {
            return $t->getType() !== TokenType::Whitespace;
        });

        // Return the array values only, because array_filter preserves the keys
        return array_values($filteredTokens);
    }
1152695
protected function __remove( $tokens )
	{
		$updated = false;

		foreach ( $tokens as $token ) {
			if ( $this->__contains( $token ) ) {

			if ( false !== ( $index = array_search( $token, $this->tokens, true ) ) )
				unset( $this->tokens[ $index ] );
				$updated = true;
			}
		}

		if ( $updated ) {
			$this->__update();
		}

		return $this;
	}
90838
public function getByIndex(int $index) : string
        {
                $value = $this->_redis->lindex($this->_key, $index);
                return !empty($value)?$value:'';
        }
84249
function process_token(token, index, tokens) {
	var last = (index === (tokens.length - 1));
	if (token[0] === "+") return process_single(token, last);
	else if (token[0] === "#") return process_multi(token, last);
	else return process_raw(token, last);
}
1164306
protected function isShortArraySyntax(array $tokens, Node\Expr\Array_ $array)
    {
        $i = $array->getAttribute('startTokenPos');
        return is_string($tokens[$i]);
    }
546583
public function process($tokens) {
        $prev_category = "";
        $token_category = "";
        $skip_next = 0;
        $out = false;

	// $tokens may come as a numeric indexed array starting with an index greater than 0 (or as a boolean)
	$tokenCount = count($tokens);
        if ( is_array($tokens) ){
          $tokens = array_values($tokens);
        }
        for ($tokenNumber = 0; $tokenNumber < $tokenCount; ++$tokenNumber) {

            // https://github.com/greenlion/PHP-SQL-Parser/issues/279
            // https://github.com/sinri/PHP-SQL-Parser/commit/eac592a0e19f1df6f420af3777a6d5504837faa7
            // as there is no pull request for 279 by the user. His solution works and tested.
            if (!isset($tokens[$tokenNumber])) continue;// as a fix by Sinri 20180528
            $token = $tokens[$tokenNumber];
            $trim = trim($token); // this removes also \n and \t!

            // if it starts with an "(", it should follow a SELECT
            if ($trim !== "" && $trim[0] === "(" && $token_category === "") {
                $token_category = 'BRACKET';
                $prev_category = $token_category;
            }

            /*
             * If it isn't obvious, when $skip_next is set, then we ignore the next real token, that is we ignore whitespace.
             */
            if ($skip_next > 0) {
                if ($trim === "") {
                    if ($token_category !== "") { // is this correct??
                        $out[$token_category][] = $token;
                    }
                    continue;
                }
                // to skip the token we replace it with whitespace
                $trim = "";
                $token = "";
                $skip_next--;
                if ($skip_next > 0) {
                    continue;
                }
            }

            $upper = strtoupper($trim);
            switch ($upper) {

            /* Tokens that get their own sections. These keywords have subclauses. */
            case 'SELECT':
            case 'ORDER':
            case 'VALUES':
            case 'GROUP':
            case 'HAVING':
            case 'WHERE':
            case 'CALL':
            case 'PROCEDURE':
            case 'FUNCTION':
            case 'SERVER':
            case 'LOGFILE':
            case 'DEFINER':
            case 'RETURNS':
            case 'TABLESPACE':
            case 'TRIGGER':
            case 'DO':
            case 'FLUSH':
            case 'KILL':
            case 'RESET':
            case 'STOP':
            case 'PURGE':
            case 'EXECUTE':
            case 'PREPARE':
                $token_category = $upper;
                break;

            case 'DEALLOCATE':
                if ($trim === 'DEALLOCATE') {
                    $skip_next = 1;
                }
                $token_category = $upper;
                break;

            case 'DUPLICATE':
                if ($token_category !== 'VALUES') {
                    $token_category = $upper;
                }
                break;

            case 'SET':
                if ($token_category !== 'TABLE') {
                    $token_category = $upper;
                }
                break;

            case 'LIMIT':
            case 'PLUGIN':
            // no separate section
                if ($token_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'FROM':
            // this FROM is different from FROM in other DML (not join related)
                if ($token_category === 'PREPARE') {
                    continue 2;
                }
                // no separate section
                if ($token_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'EXPLAIN':
            case 'DESCRIBE':
            case 'SHOW':
                $token_category = $upper;
                break;

            case 'DESC':
                if ($token_category === '') {
                    // short version of DESCRIBE
                    $token_category = $upper;
                }
                // else direction of ORDER-BY
                break;

            case 'RENAME':
                $token_category = $upper;
                break;

            case 'DATABASE':
            case 'SCHEMA':
                if ($prev_category === 'DROP') {
                    continue;
                }
                if ($prev_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'EVENT':
            // issue 71
                if ($prev_category === 'DROP' || $prev_category === 'ALTER' || $prev_category === 'CREATE') {
                    $token_category = $upper;
                }
                break;

            case 'DATA':
            // prevent wrong handling of DATA as keyword
                if ($prev_category === 'LOAD') {
                    $token_category = $upper;
                }
                break;

            case 'INTO':
            // prevent wrong handling of CACHE within LOAD INDEX INTO CACHE...
                if ($prev_category === 'LOAD') {
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                $token_category = $prev_category = $upper;
                break;

            case 'USER':
            // prevent wrong processing as keyword
                if ($prev_category === 'CREATE' || $prev_category === 'RENAME' || $prev_category === 'DROP') {
                    $token_category = $upper;
                }
                break;

            case 'VIEW':
            // prevent wrong processing as keyword
                if ($prev_category === 'CREATE' || $prev_category === 'ALTER' || $prev_category === 'DROP') {
                    $token_category = $upper;
                }
                break;

            /*
             * These tokens get their own section, but have no subclauses. These tokens identify the statement but have no specific subclauses of their own.
             */
            case 'DELETE':
            case 'ALTER':
            case 'INSERT':
            case 'TRUNCATE':
            case 'OPTIMIZE':
            case 'GRANT':
            case 'REVOKE':
            case 'HANDLER':
            case 'LOAD':
            case 'ROLLBACK':
            case 'SAVEPOINT':
            case 'UNLOCK':
            case 'INSTALL':
            case 'UNINSTALL':
            case 'ANALZYE':
            case 'BACKUP':
            case 'CHECKSUM':
            case 'REPAIR':
            case 'RESTORE':
            case 'HELP':
                $token_category = $upper;
                // set the category in case these get subclauses in a future version of MySQL
                $out[$upper][0] = $trim;
                continue 2;

            case 'REPLACE':
            	if ($prev_category === '') {
            		// set the category in case these get subclauses in a future version of MySQL
            		$token_category = $upper;
            		$out[$upper][0] = $trim;
            		continue 2;
            	}
                // part of the CREATE TABLE statement or a function
                $out[$prev_category][] = $trim;
                continue 2;

            case 'IGNORE':
                if ($prev_category === 'TABLE') {
                    // part of the CREATE TABLE statement
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                if ($token_category === 'FROM') {
                    // part of the FROM statement (index hint)
                    $out[$token_category][] = $trim;
                    continue 2;
                }
                $out['OPTIONS'][] = $upper;
                continue 2;

            case 'CHECK':
                if ($prev_category === 'TABLE') {
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                $token_category = $upper;
                $out[$upper][0] = $trim;
                continue 2;

            case 'CREATE':
                if ($prev_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'INDEX':
	            if ( in_array( $prev_category, array( 'CREATE', 'DROP' ) ) ) {
		            $out[ $prev_category ][] = $trim;
		            $token_category          = $upper;
	            }
	            break;

            case 'TABLE':
                if ($prev_category === 'CREATE') {
                    $out[$prev_category][] = $trim;
                    $token_category = $upper;
                }
                if ($prev_category === 'TRUNCATE') {
                    $out[$prev_category][] = $trim;
                    $token_category = $upper;
                }
                break;

            case 'TEMPORARY':
                if ($prev_category === 'CREATE') {
                    $out[$prev_category][] = $trim;
                    $token_category = $prev_category;
                    continue 2;
                }
                break;

            case 'IF':
                if ($prev_category === 'TABLE') {
                    $token_category = 'CREATE';
                    $out[$token_category] = array_merge($out[$token_category], $out[$prev_category]);
                    $out[$prev_category] = array();
                    $out[$token_category][] = $trim;
                    $prev_category = $token_category;
                    continue 2;
                }
                break;

            case 'NOT':
                if ($prev_category === 'CREATE') {
                    $token_category = $prev_category;
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                break;

            case 'EXISTS':
                if ($prev_category === 'CREATE') {
                    $out[$prev_category][] = $trim;
                    $prev_category = $token_category = 'TABLE';
                    continue 2;
                }
                break;

            case 'CACHE':
                if ($prev_category === "" || $prev_category === 'RESET' || $prev_category === 'FLUSH'
                    || $prev_category === 'LOAD') {
                    $token_category = $upper;
                    continue 2;
                }
                break;

            /* This is either LOCK TABLES or SELECT ... LOCK IN SHARE MODE */
            case 'LOCK':
                if ($token_category === "") {
                    $token_category = $upper;
                    $out[$upper][0] = $trim;
                } elseif ($token_category === 'INDEX') {
                    break;
                } else {
                    $trim = 'LOCK IN SHARE MODE';
                    $skip_next = 3;
                    $out['OPTIONS'][] = $trim;
                }
                continue 2;

            case 'USING': /* USING in FROM clause is different from USING w/ prepared statement*/
                if ($token_category === 'EXECUTE') {
                    $token_category = $upper;
                    continue 2;
                }
                if ($token_category === 'FROM' && !empty($out['DELETE'])) {
                    $token_category = $upper;
                    continue 2;
                }
                break;

            /* DROP TABLE is different from ALTER TABLE DROP ... */
            case 'DROP':
                if ($token_category !== 'ALTER') {
                    $token_category = $upper;
                    continue 2;
                }
                break;

            case 'FOR':
                if ($prev_category === 'SHOW') {
                    continue;
                }
                $skip_next = 1;
                $out['OPTIONS'][] = 'FOR UPDATE'; // TODO: this could be generate problems within the position calculator
                continue 2;

            case 'UPDATE':
                if ($token_category === "") {
                    $token_category = $upper;
                    continue 2;
                }
                if ($token_category === 'DUPLICATE') {
                    continue 2;
                }
                break;

            case 'START':
                $trim = "BEGIN";
                $out[$upper][0] = $upper; // TODO: this could be generate problems within the position calculator
                $skip_next = 1;
                break;

            // This token is ignored, except within RENAME
            case 'TO':
                if ($token_category === 'RENAME') {
                    break;
                }
                continue 2;

            // This token is ignored, except within CREATE TABLE
            case 'BY':
                if ($prev_category === 'TABLE') {
                    break;
                }
                continue 2;

            // These tokens are ignored.
            case 'ALL':
            case 'SHARE':
            case 'MODE':
            case ';':
                continue 2;

            case 'KEY':
                if ($token_category === 'DUPLICATE') {
                    continue 2;
                }
                break;

            /* These tokens set particular options for the statement. */
            case 'LOW_PRIORITY':
            case 'DELAYED':
            case 'QUICK':
            case 'HIGH_PRIORITY':
                $out['OPTIONS'][] = $trim;
                continue 2;

            case 'USE':
                if ($token_category === 'FROM') {
                    // index hint within FROM clause
                    $out[$token_category][] = $trim;
                    continue 2;
                }
                // set the category in case these get subclauses in a future version of MySQL
                $token_category = $upper;
                $out[$upper][0] = $trim;
                continue 2;

            case 'FORCE':
                if ($token_category === 'FROM') {
                    // index hint within FROM clause
                    $out[$token_category][] = $trim;
                    continue 2;
                }
                $out['OPTIONS'][] = $trim;
                continue 2;

            case 'WITH':
                if ($token_category === 'GROUP') {
                    $skip_next = 1;
                    $out['OPTIONS'][] = 'WITH ROLLUP'; // TODO: this could be generate problems within the position calculator
                    continue 2;
                }
                if ($token_category === '') {
                	$token_category = $upper;
                }
                break;

            case 'AS':
                break;

            case '':
            case ',':
            case ';':
                break;

            default:
                break;
            }

            // remove obsolete category after union (empty category because of
            // empty token before select)
            if ($token_category !== "" && ($prev_category === $token_category)) {
                $out[$token_category][] = $token;
            }

            $prev_category = $token_category;
        }

        return parent::process($out);
    }
1281120
private function isFunctionCall(Tokens $tokens, int $index): bool
    {
        $previousIndex = $tokens->getPrevMeaningfulToken($index);

        return $previousIndex !== null
            && ($tokens[$previousIndex]->isGivenKind(CT::T_FUNCTION_IMPORT)
                || $tokens[$previousIndex]->isGivenKind(T_FUNCTION));
    }
49591
private function shuntingYard(array $tokens)
    {
        // Clear the oepratorStack
        $this->operatorStack = new Stack();

        // Clear the operandStack.
        $this->operandStack = new Stack();

        // Remember the last token handled, this is done to recognize unary operators.
        $lastNode = null;

        // Loop over the tokens
        for ($index = 0; $index < count($tokens); $index++)
        {
            $token = $tokens[$index];

            // echo "current token $token\n";
            // echo("operands:" . $this->operandStack . "\n");
            // echo("operators: " . $this->operatorStack . "\n\n");

            if ($this->rationalFactory) {
                $node = Node::rationalFactory($token);
            } else {
                $node = Node::factory($token);
            }

            // Handle closing parentheses
            if ($token->getType() == TokenType::CloseParenthesis) {
                $this->handleSubExpression();
            }
            // Push terminal tokens on the operandStack
            elseif ($node->isTerminal()) {
                $this->operandStack->push($node);

                // Push function applications or open parentheses `(` onto the operatorStack
            } elseif ($node instanceof FunctionNode) {
                $this->operatorStack->push($node);

            } elseif ($node instanceof SubExpressionNode) {
                $this->operatorStack->push($node);

                // Handle the remaining operators.
            } elseif ($node instanceof PostfixOperatorNode) {
                $op = $this->operandStack->pop();
                if ($op == NULL) throw new SyntaxErrorException();
                $this->operandStack->push(new FunctionNode($node->getOperator(), $op));

            } elseif ($node instanceof ExpressionNode) {

                // Check for unary minus and unary plus.
                $unary = $this->isUnary($node, $lastNode);

                if ($unary) {
                    switch($token->getType()) {
                        // Unary +, just ignore it
                        case TokenType::AdditionOperator:
                        $node = null;
                        break;
                        // Unary -, replace the token.
                        case TokenType::SubtractionOperator:
                        $node->setOperator('~');
                        break;
                    }
                } else {
                    // Pop operators with higher priority

                    while ($node->lowerPrecedenceThan($this->operatorStack->peek())) {

                        $popped = $this->operatorStack->pop();
                        $popped = $this->handleExpression($popped);
                        $this->operandStack->push($popped);
                    }
                }

                if ($node) $this->operatorStack->push($node);
            }

            // Remember the current token (if it hasn't been nulled, for example being a unary +)
            if ($node) $lastNode = $node;
        }


        // Pop remaining operators

        while (!$this->operatorStack->isEmpty()) {
            $node = $this->operatorStack->pop();
            $node = $this->handleExpression($node);
            $this->operandStack->push($node);
        }

        // Stack should be empty here
        if ($this->operandStack->count() > 1) {
            throw new SyntaxErrorException();
        }

        return $this->operandStack->pop();
    }
475682
public static function alias(array $tokens) {
        $column     = self::column($tokens, new MetaData());
        $tableAlias = Table::alias($tokens);

        return empty($tableAlias) ? $column : $tableAlias . '.' . $column;
    }
1205663
public function hasRequiredTokens(array $tokens, array $tokenIds) {
		foreach ($tokenIds as $id) {
			if (in_array($tokens[$id]['id'], $this->requiredTokens)) {
				return true;
			}
		}
		return false;
	}
797087
private static function _arrayFindName(array $tokenAnswers, $fieldNames)
    {
        $results = array();

        foreach ($tokenAnswers as $fieldName => $value) {
            if (strpos($fieldName, $fieldNames) !== false) {
                $results[] = $fieldName;
            }
        }

        // \MUtil_Echo::track($results);

        return $results;
    }
381240
public function seek($index, $token = false)
    {
        $this->_current = (int) $index;
        return $this->current($token);
    }
118350
def fetch(index, tokens):
    
    if len(tokens) == 0:
        return set()
    return set.intersection(*[set(index.get(token, [])) for token in tokens])
348029
private static function getSegmentFromToken($token, $key, array $tokens)
    {
        $relation_tokens = array('>', '~', '+');

        if (\in_array($token, $relation_tokens, true)) { // not a segment
            return false;
        }

        $segment = (object) array(
            'selector' => '',
            'relation_token' => false,
            'attribute_filters' => array(),
            'pseudo_filters' => array()
        );

        if (isset($tokens[$key-1]) && \in_array($tokens[$key-1], $relation_tokens, true)) { // get relationship token
            $segment->relation_token = $tokens[$key-1];
        }

        if (ctype_alpha($token)) { // simple element selector
            $segment->selector = $token;
            return $segment;
        }

        $char_tmp_replaced = false;
        if (strpos($token, '\\') !== false) {
            $token = preg_replace_callback( // temporary replace escaped characters
                '#(\\\\)(.{1})#',
                function ($matches) {
                    return 'ESCAPED'. \ord($matches[2]);
                },
                $token
            );
            $char_tmp_replaced = true;
        }

        $segment->selector = $token;

        while (preg_match('/(.*)\:(not|contains|has)\((.+)\)$/i', $segment->selector, $matches)) { // pseudo selector
            $segment->selector = $matches[1];
            $segment->pseudo_filters[] = $matches[2].'('.$matches[3].')';
        }

        while (preg_match('/(.*)\:([a-z][a-z\-]+)$/i', $segment->selector, $matches)) { // pseudo selector
            $segment->selector = $matches[1];
            $segment->pseudo_filters[] = $matches[2];
        }

        while (preg_match('/(.*)\[([^]]+)\]$/', $segment->selector, $matches)) { // attribute selector
            $segment->selector = $matches[1];
            $segment->attribute_filters[] = $matches[2];
        }

        while (preg_match('/(.*)\.([^\.\#]+)$/i', $segment->selector, $matches)) { // class selector
            $segment->selector = $matches[1];
            $segment->attribute_filters[] = 'class~="'.$matches[2].'"';
        }

        while (preg_match('/(.*)\#([^\.\#]+)$/i', $segment->selector, $matches)) { // id selector
            $segment->selector = $matches[1];
            $segment->attribute_filters[] = 'id="'.$matches[2].'"';
        }

        if ($char_tmp_replaced) { // restore temporary replaced characters
            $set_escape_back = function (string $str) {
                return preg_replace_callback(
                    '#(ESCAPED)([0-9]{1,3})#',
                    function ($matches) {
                        return \chr($matches[2]);
                    },
                    $str
                );
            };

            $segment->selector = $set_escape_back($segment->selector);

            foreach ($segment->attribute_filters as &$attr_filter) {
                $attr_filter = $set_escape_back($attr_filter);
            }
        }

        return $segment;
    }