Scans for !=-style expression.

e.g.
!= expr
= expr

Expression-tokens always have:
escaped, which indicates that the expression result should be escaped
value, which is the code of the expression

@return \Generator
547100
protected function scanExpression()
    {

        foreach ($this->scanToken(
            'expression',
            "([?]?[!]?[=])[\t ]*(?<value>[^\n]*)"
        ) as $token) {

            $token['escaped'] = strstr($this->getMatch(1), '!') ? false : true;
            $token['unchecked'] = strstr($this->getMatch(1), '?') ? true : false;
            yield $token;
        }
    }
1064592
public function stringExpression()
    {
        $lookaheadType = $this->lexer->lookahead['type'];

        switch ($lookaheadType) {
            case Lexer::T_IDENTIFIER:
                return $this->identificationVariable();

            case Lexer::T_STRING:
                $this->match(Lexer::T_STRING);
                return new LiteralNode(LiteralNode::STRING, $this->lexer->token['value']);
        }

        $this->syntaxError(
            'StateFieldPathExpression | string'
        );

        return null;
    }
1092518
public function conditionalPrimary()
    {
        $condPrimary = new ConditionalPrimaryNode();

        if ( ! $this->lexer->isNextToken(Lexer::T_OPEN_PARENTHESIS)) {
            $condPrimary->setSimpleConditionalExpression($this->simpleConditionalExpression());

            return $condPrimary;
        }

        // Peek beyond the matching closing parenthesis ')'
        $peek = $this->peekBeyondClosingParenthesis();

        if (in_array($peek['value'], array("=",  "<", "<=", "<>", ">", ">=", "!=")) ||
            in_array($peek['type'], array(Lexer::T_NOT, Lexer::T_BETWEEN, Lexer::T_LIKE, Lexer::T_IN, Lexer::T_IS))) {
            $condPrimary->setSimpleConditionalExpression($this->simpleConditionalExpression());

            return $condPrimary;
        }

        $this->match(Lexer::T_OPEN_PARENTHESIS);
        $condPrimary->setConditionalExpression($this->conditionalExpression());
        $this->match(Lexer::T_CLOSE_PARENTHESIS);

        return $condPrimary;
    }
513641
private function lexExpression()
    {
        $statementMethods = $this->getStatementLexingMethods();

        while (!$this->atEof()) {
            reset($statementMethods);
            $matchedStatement = false;

            // This is essentially a foreach loop that can be reset
            while (list($statementOpenDelimiter, $methodName) = each($statementMethods)) {
                if ($this->matches($statementOpenDelimiter)) {
                    // This is an unescaped statement
                    $matchedStatement = true;
                    $this->{$methodName}();

                    // Now that we've matched, we want to reset the loop so that longest delimiters are matched first
                    reset($statementMethods);
                } elseif ($this->getCurrentChar() === '\\') {
                    // Now that we know we're on an escape character, spend the resources to check for a match
                    if ($this->matches("\\$statementOpenDelimiter")) {
                        // This is an escaped statement
                        $this->expressionBuffer .= $statementOpenDelimiter;
                    }
                }
            }

            // Handle any text outside statements
            if (!$matchedStatement && !$this->atEof()) {
                $currentChar = $this->getCurrentChar();
                $this->expressionBuffer .= $currentChar;
                $this->cursor++;

                // Keep on going if we're seeing alphanumeric text
                while (ctype_alnum($this->getCurrentChar())) {
                    $currentChar = $this->getCurrentChar();
                    $this->expressionBuffer .= $currentChar;
                    $this->cursor++;
                }
            } else {
                $this->flushExpressionBuffer();
            }
        }

        // Flush anything left over in the buffer
        $this->flushExpressionBuffer();
    }
956558
public function nullComparisonExpression()
    {
        $expr = $this->identificationVariable();

        $nullCompExpr = new NullComparisonExpressionNode($expr);

        $this->match(Lexer::T_IS);

        if ($this->lexer->isNextToken(Lexer::T_NOT)) {
            $this->match(Lexer::T_NOT);

            $nullCompExpr->setNot(true);
        }

        $this->match(Lexer::T_NULL);

        return $nullCompExpr;
    }
1205106
public function parseExpression()
    {
        $out = '';
        $brackets = array();

        do {

            if ($token = $this->stream->nextIf(T_STRING)) {

                $name = $token->getValue();

                if ($this->stream->test('(') && $this->engine->getFunction($name)) {
                    $out .= sprintf("\$this->callFunction('%s', array%s)", $name, $this->parseExpression());
                } else {
                    $out .= $name;
                }

            } elseif ($token = $this->stream->nextIf(T_VARIABLE)) {

                $out .= $this->parseSubscript($var = $token->getValue());
                $this->variables[ltrim($var, '$')] = null;

            } else {

                $token = $this->stream->next();

                if ($token->test(array('(', '['))) {
                    array_push($brackets, $token);
                } elseif ($token->test(array(')', ']'))) {
                    array_pop($brackets);
                }

                $out .= $token->getValue();
            }

        } while (!empty($brackets));

        return $out;
    }
1177737
protected function getParamExpression() {
		if ($this->paramExpression === null) {
			$expr = sprintf(preg_quote($this->paramsPattern, '~'), '(.*?)');
			
			$this->paramExpression = '~' . $expr . '~';
		}
		
		return $this->paramExpression;
	}
538500
public function lex()
    {
        // TODO: Sometimes, static::parse* functions make unnecessary calls to
        // is* functions. For a better performance, some rules can be deduced
        // from context.
        // For example, in `parseBool` there is no need to compare the token
        // every time with `true` and `false`. The first step would be to
        // compare with 'true' only and just after that add another letter from
        // context and compare again with `false`.
        // Another example is `parseComment`.

        $list = new TokensList();

        /**
         * Last processed token.
         *
         * @var Token
         */
        $lastToken = null;

        for ($this->last = 0, $lastIdx = 0; $this->last < $this->len; $lastIdx = ++$this->last) {
            /**
             * The new token.
             *
             * @var Token
             */
            $token = null;

            foreach (static::$PARSER_METHODS as $method) {
                if ($token = $this->$method()) {
                    break;
                }
            }

            if ($token === null) {
                // @assert($this->last === $lastIdx);
                $token = new Token($this->str[$this->last]);
                $this->error(
                    'Unexpected character.',
                    $this->str[$this->last],
                    $this->last
                );
            } elseif ($lastToken !== null
                && $token->type === Token::TYPE_SYMBOL
                && $token->flags & Token::FLAG_SYMBOL_VARIABLE
                && (
                    $lastToken->type === Token::TYPE_STRING
                    || (
                        $lastToken->type === Token::TYPE_SYMBOL
                        && $lastToken->flags & Token::FLAG_SYMBOL_BACKTICK
                    )
                )
            ) {
                // Handles ```... FROM 'user'@'%' ...```.
                $lastToken->token .= $token->token;
                $lastToken->type = Token::TYPE_SYMBOL;
                $lastToken->flags = Token::FLAG_SYMBOL_USER;
                $lastToken->value .= '@' . $token->value;
                continue;
            } elseif ($lastToken !== null
                && $token->type === Token::TYPE_KEYWORD
                && $lastToken->type === Token::TYPE_OPERATOR
                && $lastToken->value === '.'
            ) {
                // Handles ```... tbl.FROM ...```. In this case, FROM is not
                // a reserved word.
                $token->type = Token::TYPE_NONE;
                $token->flags = 0;
                $token->value = $token->token;
            }

            $token->position = $lastIdx;

            $list->tokens[$list->count++] = $token;

            // Handling delimiters.
            if ($token->type === Token::TYPE_NONE && $token->value === 'DELIMITER') {
                if ($this->last + 1 >= $this->len) {
                    $this->error(
                        'Expected whitespace(s) before delimiter.',
                        '',
                        $this->last + 1
                    );
                    continue;
                }

                // Skipping last R (from `delimiteR`) and whitespaces between
                // the keyword `DELIMITER` and the actual delimiter.
                $pos = ++$this->last;
                if (($token = $this->parseWhitespace()) !== null) {
                    $token->position = $pos;
                    $list->tokens[$list->count++] = $token;
                }

                // Preparing the token that holds the new delimiter.
                if ($this->last + 1 >= $this->len) {
                    $this->error(
                        'Expected delimiter.',
                        '',
                        $this->last + 1
                    );
                    continue;
                }
                $pos = $this->last + 1;

                // Parsing the delimiter.
                $this->delimiter = null;
                $delimiterLen = 0;
                while (++$this->last < $this->len && ! Context::isWhitespace($this->str[$this->last]) && $delimiterLen < 15) {
                    $this->delimiter .= $this->str[$this->last];
                    ++$delimiterLen;
                }

                if (empty($this->delimiter)) {
                    $this->error(
                        'Expected delimiter.',
                        '',
                        $this->last
                    );
                    $this->delimiter = ';';
                }

                --$this->last;

                // Saving the delimiter and its token.
                $this->delimiterLen = strlen($this->delimiter);
                $token = new Token($this->delimiter, Token::TYPE_DELIMITER);
                $token->position = $pos;
                $list->tokens[$list->count++] = $token;
            }

            $lastToken = $token;
        }

        // Adding a final delimiter to mark the ending.
        $list->tokens[$list->count++] = new Token(null, Token::TYPE_DELIMITER);

        // Saving the tokens list.
        $this->list = $list;
    }
1193722
public function conditionalExpression()
    {
        $conditionalTerms = array();
        $conditionalTerms[] = $this->conditionalTerm();

        while ($this->lexer->isNextToken(Lexer::T_OR)) {
            $this->match(Lexer::T_OR);

            $conditionalTerms[] = $this->conditionalTerm();
        }

        // Phase 1 AST optimization: Prevent AST\ConditionalExpression
        // if only one AST\ConditionalTerm is defined
        if (count($conditionalTerms) == 1) {
            return $conditionalTerms[0];
        }

        return new ConditionalExpressionNode($conditionalTerms);
    }
1102875
private function expr($static = FALSE) {
    static $end_expression_types = [':', ';', ',', ')', ']', '}', T_AS, T_DOUBLE_ARROW, T_CLOSE_TAG];
    // Group tokens into operands & operators to pass to the expression parser
    $expression_nodes = [];
    while ($this->currentType !== NULL && !in_array($this->currentType, $end_expression_types)) {
      if ($op = $this->exprOperator($static)) {
        $expression_nodes[] = $op;
        if ($op->type === T_INSTANCEOF) {
          $expression_nodes[] = $this->classNameReference();
        }
      }
      elseif ($operand = ($static ? $this->staticOperand() : $this->exprOperand())) {
        $expression_nodes[] = $operand;
      }
      else {
        throw new ParserException(
          $this->filename,
          $this->iterator->getLineNumber(),
          $this->iterator->getColumnNumber(),
          "invalid expression");
      }
    }
    return $this->expressionParser->parse($expression_nodes, $this->filename);
  }
1182957
public function parse()
    {
        // Note: Order is very important here!
        $operators = [
            Expression::OPERATOR_GTE,
            Expression::OPERATOR_LTE,
            Expression::OPERATOR_NOT,
            Expression::OPERATOR_EQ,
            Expression::OPERATOR_GT,
            Expression::OPERATOR_LT
        ];

        foreach ($operators as $operator) {
            $pos = strpos($this->rawExpression, $operator);

            if ($pos) {

                return new Expression(
                    substr($this->rawExpression, 0, $pos),
                    $operator,
                    substr($this->rawExpression, $pos + strlen($operator))
                );
            }
        }

        // Throw an exception if an operator can't be found
        throw new InvalidExpressionException();
    }
1185450
protected function doConvert() {
		$expectOperator = true;

		// Iterate through all tokens, saving the operators and operands to a
		// stack per Dijkstra's shunting yard algorithm.
		/** @var Operator $token */
		while ( false !== ( $token = $this->nextToken() ) ) {
			// In this grammar, there are only binary operators, so every valid
			// rule string will alternate between operator and operand tokens.
			$expectOperator = !$expectOperator;

			if ( $token instanceof Expression ) {
				// Operand
				if ( $expectOperator ) {
					$token->error( 'unexpected operand' );
				}
				$this->operands[] = $token;
				continue;
			} else {
				// Operator
				if ( !$expectOperator ) {
					$token->error( 'unexpected operator' );
				}
				// Resolve higher precedence levels
				$lastOp = end( $this->operators );
				while ( $lastOp && self::$precedence[$token->name] <= self::$precedence[$lastOp->name] ) {
					$this->doOperation( $lastOp );
					array_pop( $this->operators );
					$lastOp = end( $this->operators );
				}
				$this->operators[] = $token;
			}
		}

		// Finish off the stack
		while ( $op = array_pop( $this->operators ) ) {
			$this->doOperation( $op );
		}

		// Make sure the result is sane. The first case is possible for an empty
		// string input, the second should be unreachable.
		if ( !count( $this->operands ) ) {
			$this->error( 'condition expected' );
		} elseif ( count( $this->operands ) > 1 ) {
			$this->error( 'missing operator or too many operands' );
		}

		$value = $this->operands[0];
		if ( $value->type !== 'boolean' ) {
			$this->error( 'the result must have a boolean type' );
		}

		return $this->operands[0]->rpn;
	}
183122
protected function parseCode()
    {
        $brace_level      = 0;
        $parsed_code      = '';
        $parsing_complete = false;

        // Parse the code looking for the end of the function
        /** @var $token \FunctionParser\Token */
        foreach ($this->tokenizer as $token)
        {
            /***********************************************************************************************************
             * AFTER PARSING
             *
             * After the parsing is complete, we need to make sure there are no other T_FUNCTION tokens found, which
             * would indicate a possible ambiguity in the function code we retrieved. This should only happen in
             * situations where the code is minified or poorly formatted.
             */
            if ($parsing_complete)
            {
                if ($token->is(T_FUNCTION))
                {
                    throw new \RuntimeException('Cannot parse the function; multiple, non-nested functions were defined'
                        . ' in the code block containing the desired function.');
                }
                else
                {
                    continue;
                }
            }

            /***********************************************************************************************************
             * WHILE PARSING
             *
             * Scan through the tokens (while keeping track of braces) and reconstruct the code from the parsed tokens.
             */

            // Keep track of opening and closing braces
            if ($token->isOpeningBrace())
            {
                $brace_level++;
            }
            elseif ($token->isClosingBrace())
            {
                $brace_level--;

                // Once we reach the function's closing brace, mark as complete
                if ($brace_level === 0)
                {
                    $parsing_complete = true;
                }
            }

            // Reconstruct the code token by token
            $parsed_code .= $token->code;
        }

        /*
         * If all tokens have been looked at and the closing brace was not found, then there is a
         * problem with the code defining the Closure. This should probably never happen, but just
         * in case...
         */
        if (!$parsing_complete)
        {
            // @codeCoverageIgnoreStart
            throw new \RuntimeException('Cannot parse the function because the code appeared to be invalid.');
            // @codeCoverageIgnoreEnd
        }

        return $parsed_code;
    }
204794
protected static function evaluateConcat($expr)
	{
		preg_match_all('(\'[^\']*\'|"[^"]*")', $expr, $strings);

		$value = '';
		foreach ($strings[0] as $string)
		{
			$value .= substr($string, 1, -1);
		}

		return $value;
	}
873608
public function comparisonOperator()
    {
        switch ($this->lexer->lookahead['value']) {
            case '=':
                $this->match(Lexer::T_EQUALS);

                return '=';
            case '<':
                $this->match(Lexer::T_LOWER_THAN);
                $operator = '<';

                if ($this->lexer->isNextToken(Lexer::T_EQUALS)) {
                    $this->match(Lexer::T_EQUALS);
                    $operator .= '=';
                } else if ($this->lexer->isNextToken(Lexer::T_GREATER_THAN)) {
                    $this->match(Lexer::T_GREATER_THAN);
                    $operator .= '>';
                }

                return $operator;

            case '>':
                $this->match(Lexer::T_GREATER_THAN);
                $operator = '>';

                if ($this->lexer->isNextToken(Lexer::T_EQUALS)) {
                    $this->match(Lexer::T_EQUALS);
                    $operator .= '=';
                }

                return $operator;

            case '!':
                $this->match(Lexer::T_NEGATE);
                $this->match(Lexer::T_EQUALS);

                return '<>';

            default:
                $this->syntaxError('=, <, <=, <>, >, >=, !=');
        }

        return null;
    }
512190
protected function scanSub()
    {

        //Escaped text after
        if ($this->peek(2) === '! ') {

            $this->consume();

            foreach ($this->scanText(true) as $token)
                yield $token;
        }

        if ($this->match('([!]?)\.')) {

            $this->consumeMatch();

            foreach ($this->scanTextBlock($this->getMatch(1) === '!') as $token)
                yield $token;
        }

        foreach ($this->scanExpansion() as $token)
            yield $token;
    }
565724
public function find()
    {
        foreach ($this->file->tokens as $i => $t) {
            if (TokenUtils::isTokenType($t, array(T_NEW))) {
                // +2 to consume whitespace
                $c = self::findClassInNextTokens($this->file->tokens, $i + 2);
                if ($c) {
                    yield $c;
                }
            } elseif (TokenUtils::isTokenType($t, array(T_IMPLEMENTS, T_EXTENDS))) {
                // +2 to consume whitespace
                $c = self::findClassInNextTokens($this->file->tokens, $i + 2);
                if ($c) {
                    yield $c;
                }
                $j = $c->to + 1;
                while ($this->file->tokens[$j] === ',') {
                    // +2 to consume comma and whitespace
                    $c = self::findClassInNextTokens($this->file->tokens, $j + 2);
                    if ($c) {
                        yield $c;
                        $j = $c->to;
                    }
                    $j++;
                }
            } elseif (TokenUtils::isTokenType($t, array(T_USE))) {
                $c = self::findClassInNextTokens($this->file->tokens, $i + 2);
                // Check if use statement is after class opening bracket
                if ($c && $this->isAfterOpeningBracket($c)) {
                    yield $c;
                }
            } elseif (TokenUtils::isTokenType($t, array(T_PAAMAYIM_NEKUDOTAYIM, T_DOUBLE_COLON))) {
                $c = self::findClassInPreviousTokens($this->file->tokens, $i - 1);
                if ($c) {
                    yield $c;
                }
            } elseif (TokenUtils::isTokenType($t, array(T_CATCH))) {
                // +3 to consume brace and whitespace
                $c = self::findClassInNextTokens($this->file->tokens, $i + 3);
                if ($c) {
                    yield $c;
                }
            } elseif (TokenUtils::isTokenType($t, array(T_INSTANCEOF))) {
                $c = self::findClassInNextTokens($this->file->tokens, $i + 2);
                if ($c) {
                    yield $c;
                }
            } elseif (TokenUtils::isTokenType($t, array(T_FUNCTION))) {
                $j = $i + 4;
                $j = TokenUtils::isTokenType($this->file->tokens[$j], T_WHITESPACE) ? $j + 1 : $j;

                // +4 to consume whitespace, name, open brace
                $c = self::findClassInNextTokens($this->file->tokens, $j);
                if ($c) {
                    yield $c;
                    $j = $c->to;
                }

                while ($this->file->tokens[$j] != ")") {
                    if ($this->file->tokens[$j] == ',') {
                        // +2 to consume comma and whitespace
                        $c = self::findClassInNextTokens($this->file->tokens, $j + 2);
                        if ($c) {
                            yield $c;
                            $j = $c->to;
                        }
                    }
                    $j++;
                }

            }
        }
    }