returns the values as list

@param  array $tokens
@return array

@SuppressWarnings("PHPMD.StaticAccess")
198266
public static function values(array $tokens) {
        $values = explode(',', self::removeBrackets(self::removeSpacesBetweenComma(end($tokens['VALUES'])['base_expr'])));

        return array_map(function($value) {
            return Value::create($value);
        }, $values);
    }
57378
public function getTokens(): array
    {
        $response = $this
            ->httpClient
            ->get(
                sprintf(
                    '/%s/tokens',
                    $this->getAppUUID()->getId()
                ),
                'get',
                [],
                [],
                Http::getApisearchHeaders($this)
            );

        self::throwTransportableExceptionIfNeeded($response);

        return array_map(function (array $token) {
            return Token::createFromArray($token);
        }, $response['body']);
    }
343434
private function parseUseStatement(\ArrayIterator $tokens)
    {
        $uses = [];
        $continue = true;

        while ($continue) {
            $this->skipToNextStringOrNamespaceSeparator($tokens);

            $uses = array_merge($uses, $this->extractUseStatements($tokens));
            if ($tokens->current()[0] === self::T_LITERAL_END_OF_USE) {
                $continue = false;
            }
        }

        return $uses;
    }
116001
public static function transpileMatchRegex(array $tokens)
    {
        $regex = [];

        foreach ($tokens as $token) {
            $var = $token instanceof Variable ? $token : ($token instanceof Delimiter ? $token->next : null);

            if (null !== $var && $var instanceof Variable && null !== ($optgrp = self::makeOptGrp($var))) {
                $regex[] = $optgrp;
                break;
            }

            $regex[] = $token;
        }

        return implode('', $regex);
    }
694202
protected function countGroupOpen(Array $tokens)
	{
		return array_reduce($tokens, function ($carry, $item) {
			return $carry += $item instanceof Token\Group\Open;
		}, 0);
	}
412841
public static function columns(array $tokens) {
        $columns = array_filter($tokens['INSERT'], function($token) {
            return $token['expr_type'] === 'column-list';
        });

        return array_map(function($column) {
            return end($column['no_quotes']['parts']);
        }, end($columns)['sub_tree']);
    }
438840
protected function getValueOfToken(array $token)
    {
        switch ($token[Lexer::TYPE]) {
            case Lexer::T_QUOTED_STRING:
                return substr($token[Lexer::VALUE], 1, -1);

            case Lexer::T_NUMBER:
                return (float)$token[Lexer::VALUE];

            case Lexer::T_TRUE:
                return true;

            case Lexer::T_FALSE:
                return false;

            case Lexer::T_NULL:
                return null;
        }

        return null;
    }
628975
public static function getValueAsList($value) {
		$tokens = token_get_all('<?php '.$value);

		$resultArray = array();
		
		// Le's find all the strings, and let's put each string in the table.
		foreach ($tokens as $token) {
			if ($token[0] == T_CONSTANT_ENCAPSED_STRING) {
				$resultArray[] = substr($token[1], 1, strlen($token[1])-2) ;
			}
		}

		return  $resultArray;
	}
654417
public function replaceTokens($tokens){
        $keys = array_keys($tokens);
        $values = array_values($tokens);

        return str_replace($keys, $values, $this->getExpression());
    }
176040
public static function serialize(array $tokens)
	{
		$attrValue = '';
		foreach ($tokens as $token)
		{
			if ($token[0] === 'literal')
			{
				$attrValue .= preg_replace('([{}])', '$0$0', $token[1]);
			}
			elseif ($token[0] === 'expression')
			{
				$attrValue .= '{' . $token[1] . '}';
			}
			else
			{
				throw new RuntimeException('Unknown token type');
			}
		}

		return $attrValue;
	}
1060705
private function parse($tokens, ?TypeContext $context = null): array
    {
        $count = count($tokens);
        $tagCount = 0;
        $tags = [];

        for ($i = 1; $i < $count; $i += 2) {
            $tags[] = $this->tagFactory->create($tokens[$i], $context);
            $tokens[$i] = '%' . ++$tagCount . '$s';
        }

        //In order to allow "literal" inline tags, the otherwise invalid
        //sequence "{@}" is changed to "@", and "{}" is changed to "}".
        //"%" is escaped to "%%" because of vsprintf.
        //See unit tests for examples.
        for ($i = 0; $i < $count; $i += 2) {
            $tokens[$i] = str_replace(['{@}', '{}', '%'], ['@', '}', '%%'], $tokens[$i]);
        }

        return [implode('', $tokens), $tags];
    }
74682
public function apply(array &$tokens)
		{
		$r = 0;
		for ($i = 0, $l = count($tokens); $i < $l; $i++)
			{
			$current	= get_class($tokens[$i]);
			$next		= isset($tokens[$i + 1]) ? get_class($tokens[$i + 1]) : false;
			if (($current === "CssRulesetStartToken" && $next === "CssRulesetEndToken") ||
				($current === "CssAtKeyframesRulesetStartToken" && $next === "CssAtKeyframesRulesetEndToken" && !array_intersect(array("from", "0%", "to", "100%"), array_map("strtolower", $tokens[$i]->Selectors)))
				)
				{
				$tokens[$i]		= null;
				$tokens[$i + 1]	= null;
				$i++;
				$r = $r + 2;
				}
			}
		return $r;
		}
546583
public function process($tokens) {
        $prev_category = "";
        $token_category = "";
        $skip_next = 0;
        $out = false;

	// $tokens may come as a numeric indexed array starting with an index greater than 0 (or as a boolean)
	$tokenCount = count($tokens);
        if ( is_array($tokens) ){
          $tokens = array_values($tokens);
        }
        for ($tokenNumber = 0; $tokenNumber < $tokenCount; ++$tokenNumber) {

            // https://github.com/greenlion/PHP-SQL-Parser/issues/279
            // https://github.com/sinri/PHP-SQL-Parser/commit/eac592a0e19f1df6f420af3777a6d5504837faa7
            // as there is no pull request for 279 by the user. His solution works and tested.
            if (!isset($tokens[$tokenNumber])) continue;// as a fix by Sinri 20180528
            $token = $tokens[$tokenNumber];
            $trim = trim($token); // this removes also \n and \t!

            // if it starts with an "(", it should follow a SELECT
            if ($trim !== "" && $trim[0] === "(" && $token_category === "") {
                $token_category = 'BRACKET';
                $prev_category = $token_category;
            }

            /*
             * If it isn't obvious, when $skip_next is set, then we ignore the next real token, that is we ignore whitespace.
             */
            if ($skip_next > 0) {
                if ($trim === "") {
                    if ($token_category !== "") { // is this correct??
                        $out[$token_category][] = $token;
                    }
                    continue;
                }
                // to skip the token we replace it with whitespace
                $trim = "";
                $token = "";
                $skip_next--;
                if ($skip_next > 0) {
                    continue;
                }
            }

            $upper = strtoupper($trim);
            switch ($upper) {

            /* Tokens that get their own sections. These keywords have subclauses. */
            case 'SELECT':
            case 'ORDER':
            case 'VALUES':
            case 'GROUP':
            case 'HAVING':
            case 'WHERE':
            case 'CALL':
            case 'PROCEDURE':
            case 'FUNCTION':
            case 'SERVER':
            case 'LOGFILE':
            case 'DEFINER':
            case 'RETURNS':
            case 'TABLESPACE':
            case 'TRIGGER':
            case 'DO':
            case 'FLUSH':
            case 'KILL':
            case 'RESET':
            case 'STOP':
            case 'PURGE':
            case 'EXECUTE':
            case 'PREPARE':
                $token_category = $upper;
                break;

            case 'DEALLOCATE':
                if ($trim === 'DEALLOCATE') {
                    $skip_next = 1;
                }
                $token_category = $upper;
                break;

            case 'DUPLICATE':
                if ($token_category !== 'VALUES') {
                    $token_category = $upper;
                }
                break;

            case 'SET':
                if ($token_category !== 'TABLE') {
                    $token_category = $upper;
                }
                break;

            case 'LIMIT':
            case 'PLUGIN':
            // no separate section
                if ($token_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'FROM':
            // this FROM is different from FROM in other DML (not join related)
                if ($token_category === 'PREPARE') {
                    continue 2;
                }
                // no separate section
                if ($token_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'EXPLAIN':
            case 'DESCRIBE':
            case 'SHOW':
                $token_category = $upper;
                break;

            case 'DESC':
                if ($token_category === '') {
                    // short version of DESCRIBE
                    $token_category = $upper;
                }
                // else direction of ORDER-BY
                break;

            case 'RENAME':
                $token_category = $upper;
                break;

            case 'DATABASE':
            case 'SCHEMA':
                if ($prev_category === 'DROP') {
                    continue;
                }
                if ($prev_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'EVENT':
            // issue 71
                if ($prev_category === 'DROP' || $prev_category === 'ALTER' || $prev_category === 'CREATE') {
                    $token_category = $upper;
                }
                break;

            case 'DATA':
            // prevent wrong handling of DATA as keyword
                if ($prev_category === 'LOAD') {
                    $token_category = $upper;
                }
                break;

            case 'INTO':
            // prevent wrong handling of CACHE within LOAD INDEX INTO CACHE...
                if ($prev_category === 'LOAD') {
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                $token_category = $prev_category = $upper;
                break;

            case 'USER':
            // prevent wrong processing as keyword
                if ($prev_category === 'CREATE' || $prev_category === 'RENAME' || $prev_category === 'DROP') {
                    $token_category = $upper;
                }
                break;

            case 'VIEW':
            // prevent wrong processing as keyword
                if ($prev_category === 'CREATE' || $prev_category === 'ALTER' || $prev_category === 'DROP') {
                    $token_category = $upper;
                }
                break;

            /*
             * These tokens get their own section, but have no subclauses. These tokens identify the statement but have no specific subclauses of their own.
             */
            case 'DELETE':
            case 'ALTER':
            case 'INSERT':
            case 'TRUNCATE':
            case 'OPTIMIZE':
            case 'GRANT':
            case 'REVOKE':
            case 'HANDLER':
            case 'LOAD':
            case 'ROLLBACK':
            case 'SAVEPOINT':
            case 'UNLOCK':
            case 'INSTALL':
            case 'UNINSTALL':
            case 'ANALZYE':
            case 'BACKUP':
            case 'CHECKSUM':
            case 'REPAIR':
            case 'RESTORE':
            case 'HELP':
                $token_category = $upper;
                // set the category in case these get subclauses in a future version of MySQL
                $out[$upper][0] = $trim;
                continue 2;

            case 'REPLACE':
            	if ($prev_category === '') {
            		// set the category in case these get subclauses in a future version of MySQL
            		$token_category = $upper;
            		$out[$upper][0] = $trim;
            		continue 2;
            	}
                // part of the CREATE TABLE statement or a function
                $out[$prev_category][] = $trim;
                continue 2;

            case 'IGNORE':
                if ($prev_category === 'TABLE') {
                    // part of the CREATE TABLE statement
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                if ($token_category === 'FROM') {
                    // part of the FROM statement (index hint)
                    $out[$token_category][] = $trim;
                    continue 2;
                }
                $out['OPTIONS'][] = $upper;
                continue 2;

            case 'CHECK':
                if ($prev_category === 'TABLE') {
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                $token_category = $upper;
                $out[$upper][0] = $trim;
                continue 2;

            case 'CREATE':
                if ($prev_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'INDEX':
	            if ( in_array( $prev_category, array( 'CREATE', 'DROP' ) ) ) {
		            $out[ $prev_category ][] = $trim;
		            $token_category          = $upper;
	            }
	            break;

            case 'TABLE':
                if ($prev_category === 'CREATE') {
                    $out[$prev_category][] = $trim;
                    $token_category = $upper;
                }
                if ($prev_category === 'TRUNCATE') {
                    $out[$prev_category][] = $trim;
                    $token_category = $upper;
                }
                break;

            case 'TEMPORARY':
                if ($prev_category === 'CREATE') {
                    $out[$prev_category][] = $trim;
                    $token_category = $prev_category;
                    continue 2;
                }
                break;

            case 'IF':
                if ($prev_category === 'TABLE') {
                    $token_category = 'CREATE';
                    $out[$token_category] = array_merge($out[$token_category], $out[$prev_category]);
                    $out[$prev_category] = array();
                    $out[$token_category][] = $trim;
                    $prev_category = $token_category;
                    continue 2;
                }
                break;

            case 'NOT':
                if ($prev_category === 'CREATE') {
                    $token_category = $prev_category;
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                break;

            case 'EXISTS':
                if ($prev_category === 'CREATE') {
                    $out[$prev_category][] = $trim;
                    $prev_category = $token_category = 'TABLE';
                    continue 2;
                }
                break;

            case 'CACHE':
                if ($prev_category === "" || $prev_category === 'RESET' || $prev_category === 'FLUSH'
                    || $prev_category === 'LOAD') {
                    $token_category = $upper;
                    continue 2;
                }
                break;

            /* This is either LOCK TABLES or SELECT ... LOCK IN SHARE MODE */
            case 'LOCK':
                if ($token_category === "") {
                    $token_category = $upper;
                    $out[$upper][0] = $trim;
                } elseif ($token_category === 'INDEX') {
                    break;
                } else {
                    $trim = 'LOCK IN SHARE MODE';
                    $skip_next = 3;
                    $out['OPTIONS'][] = $trim;
                }
                continue 2;

            case 'USING': /* USING in FROM clause is different from USING w/ prepared statement*/
                if ($token_category === 'EXECUTE') {
                    $token_category = $upper;
                    continue 2;
                }
                if ($token_category === 'FROM' && !empty($out['DELETE'])) {
                    $token_category = $upper;
                    continue 2;
                }
                break;

            /* DROP TABLE is different from ALTER TABLE DROP ... */
            case 'DROP':
                if ($token_category !== 'ALTER') {
                    $token_category = $upper;
                    continue 2;
                }
                break;

            case 'FOR':
                if ($prev_category === 'SHOW') {
                    continue;
                }
                $skip_next = 1;
                $out['OPTIONS'][] = 'FOR UPDATE'; // TODO: this could be generate problems within the position calculator
                continue 2;

            case 'UPDATE':
                if ($token_category === "") {
                    $token_category = $upper;
                    continue 2;
                }
                if ($token_category === 'DUPLICATE') {
                    continue 2;
                }
                break;

            case 'START':
                $trim = "BEGIN";
                $out[$upper][0] = $upper; // TODO: this could be generate problems within the position calculator
                $skip_next = 1;
                break;

            // This token is ignored, except within RENAME
            case 'TO':
                if ($token_category === 'RENAME') {
                    break;
                }
                continue 2;

            // This token is ignored, except within CREATE TABLE
            case 'BY':
                if ($prev_category === 'TABLE') {
                    break;
                }
                continue 2;

            // These tokens are ignored.
            case 'ALL':
            case 'SHARE':
            case 'MODE':
            case ';':
                continue 2;

            case 'KEY':
                if ($token_category === 'DUPLICATE') {
                    continue 2;
                }
                break;

            /* These tokens set particular options for the statement. */
            case 'LOW_PRIORITY':
            case 'DELAYED':
            case 'QUICK':
            case 'HIGH_PRIORITY':
                $out['OPTIONS'][] = $trim;
                continue 2;

            case 'USE':
                if ($token_category === 'FROM') {
                    // index hint within FROM clause
                    $out[$token_category][] = $trim;
                    continue 2;
                }
                // set the category in case these get subclauses in a future version of MySQL
                $token_category = $upper;
                $out[$upper][0] = $trim;
                continue 2;

            case 'FORCE':
                if ($token_category === 'FROM') {
                    // index hint within FROM clause
                    $out[$token_category][] = $trim;
                    continue 2;
                }
                $out['OPTIONS'][] = $trim;
                continue 2;

            case 'WITH':
                if ($token_category === 'GROUP') {
                    $skip_next = 1;
                    $out['OPTIONS'][] = 'WITH ROLLUP'; // TODO: this could be generate problems within the position calculator
                    continue 2;
                }
                if ($token_category === '') {
                	$token_category = $upper;
                }
                break;

            case 'AS':
                break;

            case '':
            case ',':
            case ';':
                break;

            default:
                break;
            }

            // remove obsolete category after union (empty category because of
            // empty token before select)
            if ($token_category !== "" && ($prev_category === $token_category)) {
                $out[$token_category][] = $token;
            }

            $prev_category = $token_category;
        }

        return parent::process($out);
    }
1257439
public function setTokens(array $tokens = [])
    {
        $session = \PHPSession::singleton();
        $session->setAttribute(self::TOKEN_KEY, $tokens);
    }
741738
protected function getInput(array $tokens)
    {
        $var = '';
        $firstToken = \array_pop($tokens);
        if (self::tokenIs($firstToken, self::T_STRING)) {
            $var = $firstToken[1];
        }

        return $var;
    }
448539
public function parse(array $tokens)
    {
        // Filter away any whitespace
        $tokens = $this->filterTokens($tokens);

        // Insert missing implicit multiplication tokens
        if (static::allowImplicitMultiplication()) {
            $tokens = $this->parseImplicitMultiplication($tokens);
        }

        $this->tokens = $tokens;

        // Perform the actual parsing
        return $this->shuntingYard($tokens);
    }
161809
public static function create(array $tokens) {
        HashMap::assert($tokens, 'tokens');

        if (empty($tokens['LIMIT'])) return [];

        $headers = [];
        array_push($headers, 'Limit: ' . $tokens['LIMIT']['rowcount']);
        if(empty($tokens['LIMIT']['offset'])) return $headers;
        array_push($headers, 'Offset: ' . $tokens['LIMIT']['offset']);
        return $headers;
    }