@param $expression

@return Token[]
556045
public function tokenize($expression)
    {
        if ($expression !== '') {
            $flags          = PREG_SPLIT_DELIM_CAPTURE | PREG_SPLIT_NO_EMPTY;
            $this->iterator = new \CallbackFilterIterator(
                new \ArrayIterator(
                    preg_split(self::$expressionPartsPattern, $expression, 0, $flags)
                ),
                function ($value) {
                    //skip spaces
                    if ($value === ' ') {
                        return false;
                    }

                    //skip whitespaces but count newlines
                    if (rtrim($value) === '') {
                        $this->line += substr_count($value, "\n");

                        return false;
                    }

                    return true;
                }
            );
            $this->iterator->rewind();
        } else {
            $this->iterator = new \EmptyIterator();
        }

        return new Stream($this);
    }
489305
private static function transformCssPseudoSelector($expression, array &$new_path_tokens)
    {
        if (preg_match('|not\((.+)\)|i', $expression, $matches)) {
            $parts = explode(',', $matches[1]);
            foreach ($parts as &$part) {
                $part = trim($part);
                $part = 'self::'.ltrim(self::transform($part), '/');
            }
            $not_selector = implode(' or ', $parts);
            return '[not('.$not_selector.')]';
        } elseif (preg_match('|contains\((.+)\)|i', $expression, $matches)) {
            return '[text()[contains(.,\''.$matches[1].'\')]]'; // contain the specified text
        } elseif (preg_match('|has\((.+)\)|i', $expression, $matches)) {
            if (strpos($matches[1], '> ') === 0) {
                return '[child::' . ltrim(self::transform($matches[1]), '/') .']';
            } else {
                return '[descendant::' . ltrim(self::transform($matches[1]), '/') .']';
            }
        } elseif ($expression === 'first' || $expression === 'last') { // new path inside selection
            array_unshift($new_path_tokens, '(');
            $new_path_tokens[] = ')';
        }

        //  static replacement

        $pseudo_class_selectors = array(
            'disabled' => '[@disabled]',
            'first-child' => '[not(preceding-sibling::*)]',
            'last-child' => '[not(following-sibling::*)]',
            'only-child' => '[not(preceding-sibling::*) and not(following-sibling::*)]',
            'empty' => '[count(*) = 0 and string-length() = 0]',
            'not-empty' => '[count(*) > 0 or string-length() > 0]',
            'parent' => '[count(*) > 0]',
            'header' => '[self::h1 or self::h2 or self::h3 or self::h5 or self::h5 or self::h6]',
            'odd' => '[position() mod 2 = 0]',
            'even' => '[position() mod 2 = 1]',
            'first' => '[1]',
            'last' => '[last()]',
            'root' => '[not(parent::*)]'
        );

        if (!isset($pseudo_class_selectors[$expression])) {
            throw new \Exception('Pseudo class '.$expression.' unknown');
        }

        return $pseudo_class_selectors[$expression];
    }
114864
protected function compileLoopStack($expression)
    {
        foreach ($this->getLoopStack() as $token) {
            $out = $this->{'compile'.$token}($expression);

            if (strlen($out) !== 0) {
                $expression = $out;
            }
        }

        return $expression;
    }
725020
protected function toTokens(string $string = null): array
    {
        $tokens = parent::toTokens(parent::getBodyAsString());

        // Try creating any parameters
        foreach ($this->parameters as $name => $value) {
            if (null !== $value) {
                // Add the semi-colon separator
                $tokens[\count($tokens) - 1] .= ';';
                $tokens = array_merge($tokens, $this->generateTokenLines(' '.$this->createParameter($name, $value)));
            }
        }

        return $tokens;
    }
1002831
protected function parseJsonArguments(string $expression) : array
    {
        // Prepara o json
        $expression = preg_replace('#\{\s*#','{', $expression);  // transoforma [ em { sem espaços
        $expression = preg_replace('#\s*\}#','}', $expression);  // transoforma ] em } sem espaços
        $expression = preg_replace('#\s*:#',':', $expression);    // remove espaçoes antes de :
        $expression = preg_replace('#:\s*#',':', $expression);    // remove espaçoes após de :
        $expression = preg_replace('#,\s*#',',', $expression);    // remove espaços antes das virgulas
        $expression = preg_replace('#\s*,#',',', $expression);    // remove espaços após as virgulas

        // Aspas
        $expression = preg_replace("#'([a-zA-Z0-9\_-]*)':#",'"$1":', $expression); // transforma 'xxx': em "xxx":
        $expression = preg_replace("#:'([a-zA-Z0-9\_-]*)'#",':"$1"', $expression); // transforma :'xxx' em :"xxx":
        $expression = preg_replace('#:([^"])#',':"$1', $expression);  // Adiciona aspas ausentes no inicio dos valores
        $expression = preg_replace('#([^"]),"#','$1","', $expression);// Adiciona aspas ausentes no final dos valores
        $expression = preg_replace('#([^"])}#','$1"}', $expression);  // Adiciona aspas ausentes no final do json

        $decoded = json_decode($expression, true);

        if (json_last_error() !== JSON_ERROR_NONE) {
            throw new \InvalidArgumentException('Invalid json syntax');
        }

        return $decoded;
    }
975920
private function decodeToken(string $token): array
    {
        try {
            $decoded = JWT::decode(
                $token,
                $this->options["secret"],
                (array) $this->options["algorithm"]
            );
            return (array) $decoded;
        } catch (Exception $exception) {
            $this->log(LogLevel::WARNING, $exception->getMessage(), [$token]);
            throw $exception;
        }
    }
1186514
private function query(string $expression): \DOMNodeList
    {
        return $this->xpath->query(
            $this->expressionTranslator->convertToXpath($expression)
        );
    }
207524
private function getTokens($content)
    {
        // fetch non-empty lines
        $lines = preg_split('/\r\n|\r|\n/', $content, -1, PREG_SPLIT_NO_EMPTY);
        $tokens = array();
        foreach ($lines as $line) {
            // throwing tokens after '#' sign
            $commentCharPosition = strpos($line, "#");
            $lineWithoutComments = ($commentCharPosition === false) ? $line : substr($line, 0, strpos($line, "#"));
            // storing current line tokens
            if ($lineWithoutComments) {
                $tokens = array_merge($tokens, preg_split('/\s+/', $lineWithoutComments, -1, PREG_SPLIT_NO_EMPTY));
            }
        }
        return $tokens;
    }
741934
protected function tokenizeXPathForExport($expr)
	{
		$tokens = [];
		foreach ($this->matchXPathForExport($expr) as $match)
		{
			foreach (array_reverse($match) as $k => $v)
			{
				// Use the last non-numeric match
				if (!is_numeric($k))
				{
					$tokens[] = [$k, $v];
					break;
				}
			}
		}

		return $tokens;
	}
617988
private function getTokens(string $source): array
    {
        $tokens = [];
        $line = 0;

        foreach (token_get_all($source) as $token) {
            if (isset($token[2])) {
                $line = $token[2];
            }

            if (!is_array($token)) {
                $token = [$token, $token, $line];
            }

            $tokens[] = $token;
        }

        return $tokens;
    }
58542
public static function tokenize($buffer)
    {
        if ($buffer === null) {
            return [];
        } elseif (!is_string($buffer)) {
            throw new InvalidArgumentException('Buffer must be a string.');
        }

        $tokenList = [];
        $tokenLine = 1;
        $tokenColumn = 1;

        foreach (token_get_all($buffer) as $item) {
            list($tokenName, $tokenData) = self::splitToken($item);

            $tokenList[] = new Token($tokenName, $tokenData, $tokenLine, $tokenColumn);

            $pregMatch = preg_match_all(
                '/(\r\n|\n|\r)/',
                $tokenData,
                $eolMatches,
                \PREG_OFFSET_CAPTURE
            );

            if ($pregMatch) {
                $lineCount = count($eolMatches[0]);
                $lastEolPosition = end($eolMatches[1])[1];
                $lastEolLength = strlen(end($eolMatches[1])[0]);

                $tokenLine += $lineCount;
                $tokenColumn = strlen($tokenData) - ($lastEolPosition + $lastEolLength) + 1;
            } else {
                $tokenColumn += strlen($tokenData);
            }
        }

        return new TokenListIterator($tokenList);
    }
250853
public static function filter(array $collection, $expression, array $context = [])
    {
        $language = new ExpressionLanguage();

        $result = [];
        foreach ($collection as $key => $item) {
            if ($language->evaluate($expression, array_merge($context, ['item' => $item, 'key' => $key]))) {
                $result[$key] = $item;
            }
        }

        return $result;
    }
808382
public function setExpression(Expression $expression)
    {
        $this->map['expressions']->set($expression->hashCode(), $expression->setBuilder($this));
        
        return $this;
    }
949636
private function parseExpression($expression)
    {
        $result = array();

        if (isset(self::$operatorHash[$expression[0]])) {
            $result['operator'] = $expression[0];
            $expression = substr($expression, 1);
        } else {
            $result['operator'] = '';
        }

        foreach (explode(',', $expression) as $value) {
            $value = trim($value);
            $varspec = array();
            if ($colonPos = strpos($value, ':')) {
                $varspec['value'] = substr($value, 0, $colonPos);
                $varspec['modifier'] = ':';
                $varspec['position'] = (int) substr($value, $colonPos + 1);
            } elseif (substr($value, -1) == '*') {
                $varspec['modifier'] = '*';
                $varspec['value'] = substr($value, 0, -1);
            } else {
                $varspec['value'] = (string) $value;
                $varspec['modifier'] = '';
            }
            $result['values'][] = $varspec;
        }

        return $result;
    }
452928
public function process($expression) {
        $tokens = $this->splitSQLIntoTokens($expression);
        $token_count = count($tokens);
        if ($token_count === 0) {
            return null;
        }

        /*
         * Determine if there is an explicit alias after the AS clause.
         * If AS is found, then the next non-whitespace token is captured as the alias.
         * The tokens after (and including) the AS are removed.
         */
        $base_expr = "";
        $stripped = array();
        $capture = false;
        $alias = false;
        $processed = false;

        for ($i = 0; $i < $token_count; ++$i) {
            $token = $tokens[$i];
            $upper = strtoupper($token);

            if ($upper === 'AS') {
                $alias = array('as' => true, "name" => "", "base_expr" => $token);
                $tokens[$i] = "";
                $capture = true;
                continue;
            }

            if (!$this->isWhitespaceToken($upper)) {
                $stripped[] = $token;
            }

            // we have an explicit AS, next one can be the alias
            // but also a comment!
            if ($capture) {
                if (!$this->isWhitespaceToken($upper) && !$this->isCommentToken($upper)) {
                    $alias['name'] .= $token;
                    array_pop($stripped);
                }
                $alias['base_expr'] .= $token;
                $tokens[$i] = "";
                continue;
            }

            $base_expr .= $token;
        }

        if ($alias) {
            // remove quotation from the alias
            $alias['no_quotes'] = $this->revokeQuotation($alias['name']);
            $alias['name'] = trim($alias['name']);
            $alias['base_expr'] = trim($alias['base_expr']);
        }

        $stripped = $this->processExpressionList($stripped);

        // TODO: the last part can also be a comment, don't use array_pop

        // we remove the last token, if it is a colref,
        // it can be an alias without an AS
        $last = array_pop($stripped);
        if (!$alias && $this->isColumnReference($last)) {

            // TODO: it can be a comment, don't use array_pop

            // check the token before the colref
            $prev = array_pop($stripped);

            if ($this->isReserved($prev) || $this->isConstant($prev) || $this->isAggregateFunction($prev)
                    || $this->isFunction($prev) || $this->isExpression($prev) || $this->isSubQuery($prev)
                    || $this->isColumnReference($prev) || $this->isBracketExpression($prev)|| $this->isCustomFunction($prev)) {

                $alias = array('as' => false, 'name' => trim($last['base_expr']),
                               'no_quotes' => $this->revokeQuotation($last['base_expr']),
                               'base_expr' => trim($last['base_expr']));
                // remove the last token
                array_pop($tokens);
            }
        }

        $base_expr = $expression;

        // TODO: this is always done with $stripped, how we do it twice?
        $processed = $this->processExpressionList($tokens);

        // if there is only one part, we copy the expr_type
        // in all other cases we use "EXPRESSION" as global type
        $type = ExpressionType::EXPRESSION;
        if (count($processed) === 1) {
            if (!$this->isSubQuery($processed[0])) {
                $type = $processed[0]['expr_type'];
                $base_expr = $processed[0]['base_expr'];
                $no_quotes = isset($processed[0]['no_quotes']) ? $processed[0]['no_quotes'] : null;
                $processed = $processed[0]['sub_tree']; // it can be FALSE
            }
        }

        $result = array();
        $result['expr_type'] = $type;
        $result['alias'] = $alias;
        $result['base_expr'] = trim($base_expr);
        if (!empty($no_quotes)) {
            $result['no_quotes'] = $no_quotes;
        }
        $result['sub_tree'] = (empty($processed) ? false : $processed);
        return $result;
    }
1045219
protected function compileExtendsStack($expression)
    {
        foreach (['Block', 'EndBlock', 'Include', 'Inject', 'Extends'] as $token) {
            $out = $this->{'compile'.$token}($expression);

            if (strlen($out) !== 0) {
                $expression = $out;
            }
        }

        return $expression;
    }
868467
protected function ensureArray($expression)
    {
        if (is_array($expression)) {
            return array_map([$this, 'ensureArray'], $expression);
        } elseif ($expression instanceof self) {
            return $expression->getExpression();
        }

        return $expression;
    }