Lexer code for PHP tags

This code has been moved from lexer here fo easier debugging and maintenance

@param $lex
557686
public function parsePhp($lex)
    {
        $lex->token = Smarty_Internal_Templateparser::TP_PHP;
        $close = 0;
        $lex->taglineno = $lex->line;
        $closeTag = '?>';
        if (strpos($lex->value, '<?xml') === 0) {
            $lex->is_xml = true;
            $lex->token = Smarty_Internal_Templateparser::TP_NOCACHE;
            return;
        } elseif (strpos($lex->value, '<?') === 0) {
            $lex->phpType = 'php';
        } elseif (strpos($lex->value, '<%') === 0) {
            $lex->phpType = 'asp';
            $closeTag = '%>';
        } elseif (strpos($lex->value, '%>') === 0) {
            $lex->phpType = 'unmatched';
        } elseif (strpos($lex->value, '?>') === 0) {
            if ($lex->is_xml) {
                $lex->is_xml = false;
                $lex->token = Smarty_Internal_Templateparser::TP_NOCACHE;
                return;
            }
            $lex->phpType = 'unmatched';
        } elseif (strpos($lex->value, '<s') === 0) {
            $lex->phpType = 'script';
            $closeTag = '</script>';
        } elseif (strpos($lex->value, $lex->smarty->left_delimiter) === 0) {
            if ($lex->isAutoLiteral()) {
                $lex->token = Smarty_Internal_Templateparser::TP_TEXT;
                return;
            }
            $closeTag = "{$lex->smarty->left_delimiter}/php{$lex->smarty->right_delimiter}";
            if ($lex->value == $closeTag) {
                $lex->compiler->trigger_template_error("unexpected closing tag '{$closeTag}'");
            }
            $lex->phpType = 'tag';
        }
        if ($lex->phpType == 'unmatched') {
            return;
        }
        if (($lex->phpType == 'php' || $lex->phpType == 'asp') &&
            ($lex->compiler->php_handling == Smarty::PHP_PASSTHRU || $lex->compiler->php_handling == Smarty::PHP_QUOTE)
        ) {
            return;
        }
        $start = $lex->counter + strlen($lex->value);
        $body = true;
        if (preg_match('~' . preg_quote($closeTag, '~') . '~i', $lex->data, $match, PREG_OFFSET_CAPTURE, $start)) {
            $close = $match[0][1];
        } else {
            $lex->compiler->trigger_template_error("missing closing tag '{$closeTag}'");
        }
        while ($body) {
            if (preg_match('~([/][*])|([/][/][^\n]*)|(\'[^\'\\\\]*(?:\\.[^\'\\\\]*)*\')|("[^"\\\\]*(?:\\.[^"\\\\]*)*")~', $lex->data, $match, PREG_OFFSET_CAPTURE, $start)) {
                $value = $match[0][0];
                $from = $pos = $match[0][1];
                if ($pos > $close) {
                    $body = false;
                } else {
                    $start = $pos + strlen($value);
                    $phpCommentStart = $value == '/*';
                    if ($phpCommentStart) {
                        $phpCommentEnd = preg_match('~([*][/])~', $lex->data, $match, PREG_OFFSET_CAPTURE, $start);
                        if ($phpCommentEnd) {
                            $pos2 = $match[0][1];
                            $start = $pos2 + strlen($match[0][0]);
                        }
                    }
                    while ($close > $pos && $close < $start) {
                        if (preg_match('~' . preg_quote($closeTag, '~') .
                                       '~i', $lex->data, $match, PREG_OFFSET_CAPTURE, $from)) {
                            $close = $match[0][1];
                            $from = $close + strlen($match[0][0]);
                        } else {
                            $lex->compiler->trigger_template_error("missing closing tag '{$closeTag}'");
                        }
                    }
                    if ($phpCommentStart && (!$phpCommentEnd || $pos2 > $close)) {
                        $lex->taglineno = $lex->line + substr_count(substr($lex->data, $lex->counter, $start), "\n");
                        $lex->compiler->trigger_template_error("missing PHP comment closing tag '*/'");
                    }
                }
            } else {
                $body = false;
            }
        }
        $lex->value = substr($lex->data, $lex->counter, $close + strlen($closeTag) - $lex->counter);
    }
1340631
public function tokens(&$source)
    {
        // literal from source to array
        $source = \preg_replace_callback(
            "~{$this->openLiteralRegExp}(\X*?){$this->closeLiteralRegExp}~u",
            [$this, 'literal'],
            $source
        );

        // if check literal open then throw
        if (\preg_match("~{$this->openLiteralRegExp}~u", $source))
        {
            throw new Exceptions\Logic('Literal isn\'t closed');
        }

        // if check literal close then throw
        if (\preg_match("~{$this->closeLiteralRegExp}~u", $source))
        {
            throw new Exceptions\Logic('Literal isn\'t open');
        }

        // remove comments
        $source  = \preg_replace('~\{(?<q>\*)\X*?(\k<q>)\}~u', '', $source);
        $source  = \strtr($source, $this->phpTags); // remove php tags
        $lexCode = \preg_replace('~("|\'|#|\/{2}|\/\*)~u', '?>$1<?php ', $source);

        // analysis tokens
        return $this->analysis(
        // source progress with helped tokenizer
            \token_get_all('<?php' . PHP_EOL . $lexCode)
        );
    }
23298
private function matchAny(array $tokens)
    {
        if (!$this->lexer->isNextTokenAny($tokens)) {
            throw SyntaxException::expectedToken(
                implode(
                    ' or ',
                    array_map(array($this->lexer, 'getLiteral'), $tokens)
                ),
                null,
                $this->lexer
            );
        }

        return $this->lexer->moveNext();
    }
995022
public function parse(Token $token)
    {
        $lineno = $token->getLine();
        $stream = $this->parser->getStream();

        $count = null;
        $vars = new ArrayExpression([], $lineno);
        $domain = null;
        $locale = null;
        if (!$stream->test(Token::BLOCK_END_TYPE)) {
            if ($stream->test('count')) {
                // {% trans count 5 %}
                $stream->next();
                $count = $this->parser->getExpressionParser()->parseExpression();
            }

            if ($stream->test('with')) {
                // {% trans with vars %}
                $stream->next();
                $vars = $this->parser->getExpressionParser()->parseExpression();
            }

            if ($stream->test('from')) {
                // {% trans from "messages" %}
                $stream->next();
                $domain = $this->parser->getExpressionParser()->parseExpression();
            }

            if ($stream->test('into')) {
                // {% trans into "fr" %}
                $stream->next();
                $locale = $this->parser->getExpressionParser()->parseExpression();
            } elseif (!$stream->test(Token::BLOCK_END_TYPE)) {
                throw new SyntaxError('Unexpected token. Twig was looking for the "with", "from", or "into" keyword.', $stream->getCurrent()->getLine(), $stream->getSourceContext());
            }
        }

        // {% trans %}message{% endtrans %}
        $stream->expect(Token::BLOCK_END_TYPE);
        $body = $this->parser->subparse([$this, 'decideTransFork'], true);

        if (!$body instanceof TextNode && !$body instanceof AbstractExpression) {
            throw new SyntaxError('A message inside a trans tag must be a simple text.', $body->getTemplateLine(), $stream->getSourceContext());
        }

        $stream->expect(Token::BLOCK_END_TYPE);

        return new TransNode($body, $domain, $count, $vars, $locale, $lineno, $this->getTag());
    }
903493
function handleTag_code() {
    if ($this->hasParent('pre')) {
      # ignore code blocks inside <pre>
      return;
    }
    if ($this->parser->isStartTag) {
      $this->buffer();
    } else {
      $buffer = $this->unbuffer();
      # use as many backticks as needed
      preg_match_all('#`+#', $buffer, $matches);
      if (!empty($matches[0])) {
        rsort($matches[0]);

        $ticks = '`';
        while (true) {
          if (!in_array($ticks, $matches[0])) {
            break;
          }
          $ticks .= '`';
        }
      } else {
        $ticks = '`';
      }
      if ($buffer[0] == '`' || substr($buffer, -1) == '`') {
        $buffer = ' '.$buffer.' ';
      }
      $this->out($ticks.$buffer.$ticks, true);
    }
  }
772205
public function evaluate(Lexer $lexer)
    {
        switch(true) {
            case ($lexer->isNextToken(Lexer::T_SHORT_P)) :
                throw new ParserException('Property \p (Unicode Property) not supported use \x to specify unicode character or range');
            break;
            case ($lexer->isNextToken(Lexer::T_SHORT_UNICODE_X)) :
                
                $lexer->moveNext();
                if($lexer->lookahead['value'] !== '{' )  {
                    throw new ParserException('Expecting character { after \X none found');                
                }
                
                $tokens = array();
                while($lexer->moveNext() && $lexer->lookahead['value']  !== '}') {
                    
                    # check if we nested eg.{ddd{d}
                    if($lexer->lookahead['value']  === '{') {
                        throw new ParserException('Nesting hex value ranges is not allowed');
                    }
                
                    if($lexer->lookahead['value'] !== " " && ctype_xdigit($lexer->lookahead['value']) === false) {
                        throw new ParserException(sprintf('Character %s is not a hexdeciaml digit',$lexer->lookahead['value']));
                    }
                
                    $tokens[] = $lexer->lookahead['value'];
                }
                # check that current lookahead is a closing character as it's possible to iterate to end of string (i.e. lookahead === null)
                if($lexer->lookahead['value'] !== '}') {
                    throw new ParserException('Closing quantifier token `}` not found');     
                }
                
                if(count($tokens) === 0) {
                    throw new ParserException('No hex number found inside the range');
                }
                
                $number    = trim(implode('',$tokens));
                
                return Utf8::chr(hexdec($number));
                
            break;
            case ($lexer->isNextToken(Lexer::T_SHORT_X)) :
                // only allow another 2 hex characters
                $glimpse = $lexer->glimpse();
                if($glimpse['value']  === '{') {
                    throw new ParserException('Braces not supported here');
                }
                
                $tokens = array();
                $count = 2;
                while($count > 0 && $lexer->moveNext()) {
                    $tokens[] = $lexer->lookahead['value'];
                    --$count;
                }
                
                $value     = trim(implode('',$tokens));
                return Utf8::chr(hexdec($value));
            break;
            default :
                throw new ParserException('No Unicode expression to evaluate');
        }
        
    }
86167
private function tokenize($text)
    {
        $count = preg_match_all($this->lexerRegex, $text, $matches, PREG_SET_ORDER | PREG_OFFSET_CAPTURE);
        if(false === $count || preg_last_error() !== PREG_NO_ERROR) {
            throw new \RuntimeException(sprintf('PCRE failure `%s`.', preg_last_error()));
        }

        $tokens = array();
        $position = 0;

        foreach($matches as $match) {
            switch(true) {
                case -1 !== $match['string'][1]: { $token = $match['string'][0]; $type = self::TOKEN_STRING; break; }
                case -1 !== $match['ws'][1]: { $token = $match['ws'][0]; $type = self::TOKEN_WS; break; }
                case -1 !== $match['marker'][1]: { $token = $match['marker'][0]; $type = self::TOKEN_MARKER; break; }
                case -1 !== $match['delimiter'][1]: { $token = $match['delimiter'][0]; $type = self::TOKEN_DELIMITER; break; }
                case -1 !== $match['separator'][1]: { $token = $match['separator'][0]; $type = self::TOKEN_SEPARATOR; break; }
                case -1 !== $match['open'][1]: { $token = $match['open'][0]; $type = self::TOKEN_OPEN; break; }
                case -1 !== $match['close'][1]: { $token = $match['close'][0]; $type = self::TOKEN_CLOSE; break; }
                default: { throw new \RuntimeException(sprintf('Invalid token.')); }
            }
            $tokens[] = array($type, $token, $position);
            $position += mb_strlen($token, 'utf-8');
        }

        return $tokens;
    }
1297009
function Lexer(text, cfg) {
        var self = this;
        self.page = new Page(text);
        self.cursor = new Cursor();
        self.nodeFactory = this;
        this.cfg = cfg || {};
    }
340567
function lex(text) {
  let root = new StyleNode();
  root['start'] = 0;
  root['end'] = text.length
  let n = root;
  for (let i = 0, l = text.length; i < l; i++) {
    if (text[i] === OPEN_BRACE) {
      if (!n['rules']) {
        n['rules'] = [];
      }
      let p = n;
      let previous = p['rules'][p['rules'].length - 1] || null;
      n = new StyleNode();
      n['start'] = i + 1;
      n['parent'] = p;
      n['previous'] = previous;
      p['rules'].push(n);
    } else if (text[i] === CLOSE_BRACE) {
      n['end'] = i + 1;
      n = n['parent'] || root;
    }
  }
  return root;
}
997518
public function parse(Parser $parser, TokensList $list)
    {
        /**
         * Array containing all list of clauses parsed.
         * This is used to check for duplicates.
         *
         * @var array
         */
        $parsedClauses = [];

        // This may be corrected by the parser.
        $this->first = $list->idx;

        /**
         * Whether options were parsed or not.
         * For statements that do not have any options this is set to `true` by
         * default.
         *
         * @var bool
         */
        $parsedOptions = empty(static::$OPTIONS);

        for (; $list->idx < $list->count; ++$list->idx) {
            /**
             * Token parsed at this moment.
             *
             * @var Token
             */
            $token = $list->tokens[$list->idx];

            // End of statement.
            if ($token->type === Token::TYPE_DELIMITER) {
                break;
            }

            // Checking if this closing bracket is the pair for a bracket
            // outside the statement.
            if (($token->value === ')') && ($parser->brackets > 0)) {
                --$parser->brackets;
                continue;
            }

            // Only keywords are relevant here. Other parts of the query are
            // processed in the functions below.
            if ($token->type !== Token::TYPE_KEYWORD) {
                if (($token->type !== Token::TYPE_COMMENT)
                    && ($token->type !== Token::TYPE_WHITESPACE)
                ) {
                    $parser->error('Unexpected token.', $token);
                }
                continue;
            }

            // Unions are parsed by the parser because they represent more than
            // one statement.
            if (($token->keyword === 'UNION') ||
                    ($token->keyword === 'UNION ALL') ||
                    ($token->keyword === 'UNION DISTINCT') ||
                    ($token->keyword === 'EXCEPT') ||
                    ($token->keyword === 'INTERSECT')
            ) {
                break;
            }

            $lastIdx = $list->idx;

            // ON DUPLICATE KEY UPDATE ...
            // has to be parsed in parent statement (INSERT or REPLACE)
            // so look for it and break
            if ($this instanceof Statements\SelectStatement
                && $token->value === 'ON'
            ) {
                ++$list->idx; // Skip ON

                // look for ON DUPLICATE KEY UPDATE
                $first = $list->getNextOfType(Token::TYPE_KEYWORD);
                $second = $list->getNextOfType(Token::TYPE_KEYWORD);
                $third = $list->getNextOfType(Token::TYPE_KEYWORD);

                if ($first && $second && $third
                    && $first->value === 'DUPLICATE'
                    && $second->value === 'KEY'
                    && $third->value === 'UPDATE'
                ) {
                    $list->idx = $lastIdx;
                    break;
                }
            }
            $list->idx = $lastIdx;

            /**
             * The name of the class that is used for parsing.
             *
             * @var Component
             */
            $class = null;

            /**
             * The name of the field where the result of the parsing is stored.
             *
             * @var string
             */
            $field = null;

            /**
             * Parser's options.
             *
             * @var array
             */
            $options = [];

            // Looking for duplicated clauses.
            if (! empty(Parser::$KEYWORD_PARSERS[$token->value])
                || ! empty(Parser::$STATEMENT_PARSERS[$token->value])
            ) {
                if (! empty($parsedClauses[$token->value])) {
                    $parser->error(
                        'This type of clause was previously parsed.',
                        $token
                    );
                    break;
                }
                $parsedClauses[$token->value] = true;
            }

            // Checking if this is the beginning of a clause.
            if (! empty(Parser::$KEYWORD_PARSERS[$token->value]) && $list->idx < $list->count) {
                $class = Parser::$KEYWORD_PARSERS[$token->value]['class'];
                $field = Parser::$KEYWORD_PARSERS[$token->value]['field'];
                if (! empty(Parser::$KEYWORD_PARSERS[$token->value]['options'])) {
                    $options = Parser::$KEYWORD_PARSERS[$token->value]['options'];
                }
            }

            // Checking if this is the beginning of the statement.
            if (! empty(Parser::$STATEMENT_PARSERS[$token->keyword])) {
                if (! empty(static::$CLAUSES) // Undefined for some statements.
                    && empty(static::$CLAUSES[$token->value])
                ) {
                    // Some keywords (e.g. `SET`) may be the beginning of a
                    // statement and a clause.
                    // If such keyword was found and it cannot be a clause of
                    // this statement it means it is a new statement, but no
                    // delimiter was found between them.
                    $parser->error(
                        'A new statement was found, but no delimiter between it and the previous one.',
                        $token
                    );
                    break;
                }
                if (! $parsedOptions) {
                    if (empty(static::$OPTIONS[$token->value])) {
                        // Skipping keyword because if it is not a option.
                        ++$list->idx;
                    }
                    $this->options = OptionsArray::parse(
                        $parser,
                        $list,
                        static::$OPTIONS
                    );
                    $parsedOptions = true;
                }
            } elseif ($class === null) {
                if ($this instanceof Statements\SelectStatement
                    && ($token->value === 'FOR UPDATE'
                    || $token->value === 'LOCK IN SHARE MODE')
                ) {
                    // Handle special end options in Select statement
                    // See Statements\SelectStatement::$END_OPTIONS
                    $this->end_options = OptionsArray::parse(
                        $parser,
                        $list,
                        static::$END_OPTIONS
                    );
                } elseif ($this instanceof Statements\SetStatement
                    && ($token->value === 'COLLATE'
                    || $token->value === 'DEFAULT')
                ) {
                    // Handle special end options in SET statement
                    // See Statements\SetStatement::$END_OPTIONS
                    $this->end_options = OptionsArray::parse(
                        $parser,
                        $list,
                        static::$END_OPTIONS
                    );
                } else {
                    // There is no parser for this keyword and isn't the beginning
                    // of a statement (so no options) either.
                    $parser->error('Unrecognized keyword.', $token);
                    continue;
                }
            }

            $this->before($parser, $list, $token);

            // Parsing this keyword.
            if ($class !== null) {
                // We can't parse keyword at the end of statement
                if ($list->idx >= $list->count) {
                    $parser->error('Keyword at end of statement.', $token);
                    continue;
                }
                ++$list->idx; // Skipping keyword or last option.
                $this->$field = $class::parse($parser, $list, $options);
            }

            $this->after($parser, $list, $token);
        }

        // This may be corrected by the parser.
        $this->last = --$list->idx; // Go back to last used token.
    }
1304807
function(str, lex) {
  if (lex) {
    w.plugin({
      words: lex
    });
  }
  let doc = buildText(str, w);
  doc.tagger();
  return doc;
}
762914
function dump( Lex $Lex, $tab = '' ){
		$tag = $Lex->name( $this->t );
		if( $this->is_terminal() ){
			if( $this->value && $this->value !== $this->t ){
				echo $tab, '<',$tag,">\n   ", $tab, htmlspecialchars($this->value),"\n",$tab,'</',$tag,">\n";
			}
			else {
				echo $tab, htmlspecialchars($this->value),"\n";
			}
		}
		else if( ! $this->length ){
			echo $tab, '<', $tag, " />\n";
		}
		else {
			echo $tab, '<',$tag,">\n";
			foreach( $this->children as $Child ){
				$Child->dump( $Lex, "   ".$tab);
			}
			echo $tab, '</', $tag, ">\n";
		}
	}
170670
public function parse($tokens)
    {
        $parents = array();

        // flag : are we in a <pre> Tag ?
        $this->tagPreIn = false;

        /**
         * all the actions to do
         * @var Node[] $actions
         */
        $actions = array();

        // get the actions from the html tokens
        foreach ($tokens as $token) {
            if ($token->getType() === 'code') {
                $actions = array_merge($actions, $this->getTagAction($token, $parents));
            } elseif ($token->getType() === 'txt') {
                $actions = array_merge($actions, $this->getTextAction($token));
            }
        }

        // for each identified action, we have to clean up the begin and the end of the texte
        // based on tags that surround it

        // list of the tags to clean
        $tagsToClean = array(
            'page', 'page_header', 'page_footer', 'form',
            'table', 'thead', 'tfoot', 'tr', 'td', 'th', 'br',
            'div', 'hr', 'p', 'ul', 'ol', 'li',
            'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
            'bookmark', 'fieldset', 'legend',
            'draw', 'circle', 'ellipse', 'path', 'rect', 'line', 'g', 'polygon', 'polyline',
            'option'
        );

        // list of the tags to move space
        $tagsToSpace = array(
            'span', 'font', 'label',
            'strong', 'b',
            'address', 'cite', 'em', 'i', 'samp',
            'cite', 's',
            'ins', 'u',
            'big', 'small', 'sub', 'sup'
        );

        // foreach action
        $nb = count($actions);
        for ($k = 0; $k < $nb; $k++) {
            // if it is a Text
            if ($actions[$k]->getName() !== 'write') {
                continue;
            }

            // if the tag before the text is a tag to clean => ltrim on the text
            if ($k>0) {
                if (in_array($actions[$k - 1]->getName(), $tagsToClean)) {
                    $actions[$k]->setParam('txt', ltrim($actions[$k]->getParam('txt')));
                }
            }

            if ($k < $nb - 1) {
                // if the tag after the text is a tag to clean => rtrim on the text
                if (in_array($actions[$k + 1]->getName(), $tagsToClean)) {
                    $actions[$k]->setParam('txt', rtrim($actions[$k]->getParam('txt')));
                }

                // if the tag after the text is a tag with space to move => move the space to the next write
                if (in_array($actions[$k + 1]->getName(), $tagsToSpace)) {
                    if (substr($actions[$k]->getParam('txt'), -1) == ' ') {
                        $actions[$k]->setParam('txt', rtrim($actions[$k]->getParam('txt')));
                        for ($subK = $k+2; $subK < $nb; $subK++) {
                            if ($actions[$subK]->getName() === 'write') {
                                $actions[$subK]->setParam('txt', ' '.ltrim($actions[$subK]->getParam('txt')));
                                break;
                            }
                        }
                    }
                }
            }

            // if the text is empty => remove the action
            if (!strlen($actions[$k]->getParam('txt'))) {
                unset($actions[$k]);
            }
        }

        // if we are not on the level 0 => HTML validator ERROR
        if (count($parents)) {
            if (count($parents)>1) {
                $errorMsg = 'The following tags have not been closed:';
            } else {
                $errorMsg = 'The following tag has not been closed:';
            }

            $e = new HtmlParsingException($errorMsg.' '.implode(', ', $parents));
            $e->setInvalidTag($parents[0]);
            throw $e;
        }

        $this->verifyMustContain($actions, 'thead', 'tr');
        $this->verifyMustContain($actions, 'tfoot', 'tr');

        // save the actions to do
        $this->code = array_values($actions);
    }
52857
function it_should_parse_flashes_tag_with_no_specified_catalog(
        $parser, $exprParser, $stream, $body, $nodeFactory, $node, $token, $types
    )
    {
        $stream->test(\Twig_Token::NAME_TYPE, 'using')->willReturn(false, false);
        $stream->test(\Twig_Token::BLOCK_END_TYPE)->willReturn(false);

        $exprParser->parseExpression()->willReturn($types);

        $stream->expect(\Twig_Token::BLOCK_END_TYPE)->will(function() {});

        $parser->subparse(array($this, 'isEndTag'), true)->willReturn($body);

        $nodeFactory->createFlashesNode($types, null, $body, 123)->willReturn($node);

        $this->parse($token)->shouldReturn($node);
    }
347964
private function ignoreObjectBlock(Lexer $lexer) : int
    {
        do {
            $code = $lexer->lex();

            // recursively ignore nested objects
            if (self::JSON_OBJECT_START === $code) {
                $this->ignoreObjectBlock($lexer);
            }
        } while (self::JSON_OBJECT_END !== $code);

        return $lexer->lex();
    }
578917
function () {
      var property, token;
      for (property in this.lex) {
        token = this.lex[property]

        if (token.hasText) {
          token.compiledRe = this.lex.text.matcher + token.matcher;
        } else {
          token.compiledRe = token.matcher;
        }

        token.compiledRe = new RegExp(token.compiledRe, "gi");
      }
    }
10365
private function isOpenTag($token): bool
    {
        if (!is_array($token)) {
            return false;
        }

        if ($token[0] == T_ECHO && $token[1] == '<?=') {
            return true;
        }

        return $token[0] == T_OPEN_TAG || $token[0] == T_OPEN_TAG_WITH_ECHO;
    }