{@inheritdoc}

@throws LexException
540643
public function lex(string $value): array
    {
        $offset = 0;
        $valueLength = strlen($value);
        $tokens = [];

        while ($offset < $valueLength) {
            $valueFragment = substr($value, $offset);
            $fragmentTokens = $this->lexFragment($this->referenceLexer, $valueFragment);

            foreach ($fragmentTokens as $fragmentToken) {
                $tokens[] = $fragmentToken;
                $offset += strlen($fragmentToken->getValue());
            }
        }

        return $tokens;
    }
1170033
def _lex(stream):
    
    lines = enumerate(stream, 1)
    line_no = pos = 0
    try:
        while True:
            if pos == 0:
                line_no, line = next(lines)
            matches = _tdl_lex_re.finditer(line, pos)
            pos = 0  # reset; only used for multiline patterns
            for m in matches:
                gid = m.lastindex
                if gid <= 2:  # potentially multiline patterns
                    if gid == 1:  # docstring
                        s, start_line_no, line_no, line, pos = _bounded(
                            '"""', '"""', line, m.end(), line_no, lines)
                    elif gid == 2:  # comment
                        s, start_line_no, line_no, line, pos = _bounded(
                            '#|', '|#', line, m.end(), line_no, lines)
                    yield (gid, s, line_no)
                    break
                elif gid == 30:
                    raise TdlParsingError(
                        ('Syntax error:\n  {}\n {}^'
                         .format(line, ' ' * m.start())),
                        line_number=line_no)
                else:
                    # token = None
                    # if not (6 < gid < 20):
                    #     token = m.group(gid)
                    token = m.group(gid)
                    yield (gid, token, line_no)
    except StopIteration:
        pass
976053
public function init(Parser $parser, Lexer $lexer) {
        $parser->setStartRule($this->startRule);

        foreach ($this->rules as $rule) {
            $parser->addRule($rule);
        }

        $parser->setLexer($lexer);
        $lexer->setParser($parser);

        return $this;
    }
428588
protected function lexSelectionSet(): SelectionSetNode
    {
        $start = $this->lexer->getToken();

        return new SelectionSetNode(
            $this->many(
                TokenKindEnum::BRACE_L,
                [$this, 'lexSelection'],
                TokenKindEnum::BRACE_R
            ),
            $this->createLocation($start)
        );
    }
939991
private function handleUseragentGroup(Lexer $lexer) : int
    {
        $useragentPosition = 0;

        do {
            $code = $lexer->lex();

            if (self::JSON_OBJECT_START === $code) {
                $code = $this->handleUseragentBlock($lexer, $useragentPosition);
                ++$useragentPosition;
            }
        } while (self::JSON_ARRAY_END !== $code);

        return $lexer->lex();
    }
313604
public function parse($input, $file = null)
    {
        $this->languageSpecifierLine = null;
        $this->input = $input;
        $this->file = $file;
        $this->tags = array();

        try {
            $this->lexer->analyse($this->input, 'en');
        } catch (LexerException $e) {
            throw new ParserException(
                sprintf('Lexer exception "%s" thrown for file %s', $e->getMessage(), $file),
                0,
                $e
            );
        }

        $feature = null;
        while ('EOS' !== ($predicted = $this->predictTokenType())) {
            $node = $this->parseExpression();

            if (null === $node || "\n" === $node) {
                continue;
            }

            if (!$feature && $node instanceof FeatureNode) {
                $feature = $node;
                continue;
            }

            if ($feature && $node instanceof FeatureNode) {
                throw new ParserException(sprintf(
                    'Only one feature is allowed per feature file. But %s got multiple.',
                    $this->file
                ));
            }

            if (is_string($node)) {
                throw new ParserException(sprintf(
                    'Expected Feature, but got text: "%s"%s',
                    $node,
                    $this->file ? ' in file: ' . $this->file : ''
                ));
            }

            if (!$node instanceof FeatureNode) {
                throw new ParserException(sprintf(
                    'Expected Feature, but got %s on line: %d%s',
                    $node->getKeyword(),
                    $node->getLine(),
                    $this->file ? ' in file: ' . $this->file : ''
                ));
            }
        }

        return $feature;
    }
798630
private function lexDirectiveExpression()
    {
        $this->lexDirectiveName();

        $parenthesisLevel = 0;
        $newLinesAfterExpression = 0;
        $expressionBuffer = '';

        while (!$this->matches($this->directiveDelimiters[1], false) && !$this->atEof()) {
            $currentChar = $this->getCurrentChar();

            if ($currentChar === '(') {
                $expressionBuffer .= $currentChar;
                $parenthesisLevel++;
            } elseif ($currentChar === ')') {
                $parenthesisLevel--;
                $expressionBuffer .= $currentChar;
            } elseif ($currentChar === "\n") {
                if (trim($expressionBuffer) === '') {
                    $this->line++;
                } else {
                    $newLinesAfterExpression++;
                }
            } else {
                $expressionBuffer .= $currentChar;
            }

            $this->cursor++;
        }

        if ($parenthesisLevel !== 0) {
            throw new RuntimeException(
                sprintf(
                    'Unmatched parenthesis on line %d',
                    $this->line
                )
            );
        }

        $expressionBuffer = trim($expressionBuffer);
        $expressionBuffer = $this->replaceViewFunctionCalls($expressionBuffer);

        if (!empty($expressionBuffer)) {
            $this->tokens[] = new Token(TokenTypes::T_EXPRESSION, $expressionBuffer, $this->line);
        }

        $this->line += $newLinesAfterExpression;
    }
716675
public function register(PregDateLexerInterface $lexer)
    {
        $this->regexp .= $lexer->getExpression();
        $this->lexers[] = $lexer;
    }
1140443
public function lexTpl()
    {
        $source = $this->source;
        if ($source->cursor >= $source->bound) {
            $this->addError("The parser did not find the '{$source->leftDelimiter}' template start tag", $source->cursor);
            return null;
        }
        Rules::reset($source->leftDelimiter, $source->rightDelimiter);
        $this->stack = [];
        $tree = new TreeMap();
        if (Sdopx::$debug) {
            $tree->setDebugInfo($source->getDebugInfo());
        }
        $next = ['openTpl' => 1];
        do {
            $data = $this->match($next);

            if ($data == null) {
                $this->addError("Template tag syntax is incorrect", $source->cursor);
                return null;
            }
            $tag = $data['tag'];
            $source->cursor = $data['end'];
            //清除
            $item = Rules::getItem($tag);
            if (isset($item['clear'])) {
                $clear = $item['clear'];
                $flag = end($this->stack);
                while ($flag !== null && ($flag & $clear) > 0) {
                    array_pop($this->stack);
                    $flag = end($this->stack);
                }
            }
            //出栈
            if (isset($item['close'])) {
                $close = $item['close'];
                $flag = end($this->stack);
                if ($flag !== null) {
                    if (($flag & $close) == 0) {
                        $this->addError("Template tag syntax parsing error, unclosed syntax.", $source->cursor);
                        return null;
                    }
                    array_pop($this->stack);
                }
            }
            //入栈
            if (isset($item['open'])) {
                array_push($this->stack, $item['open']);
            }
            if ($tag == 'closeTpl') {
                if (count($this->stack) != 0) {
                    $this->addError("Template tag syntax parsing error, as well as unclosed syntax tags.", $source->cursor);
                    return null;
                }
                $data['node'] = null;
                $tree->push($data);
                return $tree;
            }
            $data['node'] = end($this->stack);
            $tree->push($data);
            $next = Rules::getItem($tag, 'next');
        } while (true);
    }
1366788
public function parseContent($content, $filePath)
    {
        $this->lexer->setInput($content);

        $context = new FileContext($filePath);

        try {
            while ($token = $this->lexer->getNext()) {
                $newContext = $context->enterToken($token, $content);

                if ($newContext === null) {
                    throw new TokenException('unexpected token', $token);
                }

                $context = $newContext;
            }
        } catch (TokenException $ex) {
            $at = $ex->getToken()[Lexer::AT];

            throw new MarkupException($ex->getMessage(), $filePath, $at);
        } catch (\Exception $ex) {
            throw new MarkupException($ex->getMessage());
        }

        return $context;
    }
602324
public function lex(string $value): array
    {
        foreach (self::PATTERNS as $pattern => $tokenTypeConstant) {
            if (1 === preg_match($pattern, $value, $matches)) {
                if (null === $tokenTypeConstant) {
                    throw InvalidArgumentExceptionFactory::createForInvalidExpressionLanguageToken($value);
                }

                return [new Token($matches[0], new TokenType($tokenTypeConstant))];
            }
        }

        throw ExpressionLanguageExceptionFactory::createForCouldNotLexValue($value);
    }
422950
function lexicon($path) {
		$lex=array();
		foreach ($this->languages?:explode(',',$this->fallback) as $lang)
			foreach ($this->split($path) as $dir)
				if ((is_file($file=($base=$dir.$lang).'.php') ||
					is_file($file=$base.'.php')) &&
					is_array($dict=require($file)))
					$lex+=$dict;
				elseif (is_file($file=$base.'.ini')) {
					preg_match_all(
						'/(?<=^|\n)(?:'.
							'\[(?<prefix>.+?)\]|'.
							'(?<lval>[^\h\r\n;].*?)\h*=\h*'.
							'(?<rval>(?:\\\\\h*\r?\n|.+?)*)'.
						')(?=\r?\n|$)/',
						$this->read($file),$matches,PREG_SET_ORDER);
					if ($matches) {
						$prefix='';
						foreach ($matches as $match)
							if ($match['prefix'])
								$prefix=$match['prefix'].'.';
							elseif (!array_key_exists(
								$key=$prefix.$match['lval'],$lex))
								$lex[$key]=trim(preg_replace(
									'/\\\\\h*\r?\n/','',$match['rval']));
					}
				}
		return $lex;
	}
762914
function dump( Lex $Lex, $tab = '' ){
		$tag = $Lex->name( $this->t );
		if( $this->is_terminal() ){
			if( $this->value && $this->value !== $this->t ){
				echo $tab, '<',$tag,">\n   ", $tab, htmlspecialchars($this->value),"\n",$tab,'</',$tag,">\n";
			}
			else {
				echo $tab, htmlspecialchars($this->value),"\n";
			}
		}
		else if( ! $this->length ){
			echo $tab, '<', $tag, " />\n";
		}
		else {
			echo $tab, '<',$tag,">\n";
			foreach( $this->children as $Child ){
				$Child->dump( $Lex, "   ".$tab);
			}
			echo $tab, '</', $tag, ">\n";
		}
	}
853689
def lex(self, text):
        "'"
        if not hasattr(self, 'lexer'):
            self.lexer = self._build_lexer()
        stream = self.lexer.lex(text)
        if self.options.postlex:
            return self.options.postlex.process(stream)
        return stream
1196820
private function handleDeviceBlock(Lexer $lexer, int $useragentPosition, int $childPosition) : int
    {
        $capturedKey = false;
        $sawColon    = false;

        $branchStart     = $this->getLocationCoordinates($lexer);
        $branchLocations = [];
        $branchCoverage  = [];

        do {
            $code = $lexer->lex();

            if (self::JSON_STRING === $code && false === $capturedKey) {
                $branchLocations[] = [
                    'start' => $this->getLocationCoordinates($lexer, false, '"' . $lexer->yytext . '"'),
                    'end' => $this->getLocationCoordinates($lexer, true, '"' . $lexer->yytext . '"'),
                ];
                $branchCoverage[] = $this->getCoverageCount(
                    sprintf('u%d::c%d::d%s::p', $useragentPosition, $childPosition, $lexer->yytext),
                    $this->fileCoveredIds
                );
                $capturedKey = true;
            } elseif (self::JSON_COLON === $code) {
                $sawColon = true;
            } elseif (self::JSON_STRING === $code && true === $sawColon) {
                $capturedKey = false;
            }
        } while (self::JSON_OBJECT_END !== $code);

        $branchEnd = $this->getLocationCoordinates($lexer, true);

        $this->collectBranch($branchStart, $branchEnd, $branchLocations, $branchCoverage);

        return $lexer->lex();
    }
504511
public final void mExponent() throws RecognitionException {
		try {
			// src/main/resources/org/drools/compiler/lang/DRL5Lexer.g:95:10: ( ( 'e' | 'E' ) ( '+' | '-' )? ( '0' .. '9' )+ )
			// src/main/resources/org/drools/compiler/lang/DRL5Lexer.g:95:12: ( 'e' | 'E' ) ( '+' | '-' )? ( '0' .. '9' )+
			{
			if ( input.LA(1)=='E'||input.LA(1)=='e' ) {
				input.consume();
				state.failed=false;
			}
			else {
				if (state.backtracking>0) {state.failed=true; return;}
				MismatchedSetException mse = new MismatchedSetException(null,input);
				recover(mse);
				throw mse;
			}
			// src/main/resources/org/drools/compiler/lang/DRL5Lexer.g:95:22: ( '+' | '-' )?
			int alt14=2;
			int LA14_0 = input.LA(1);
			if ( (LA14_0=='+'||LA14_0=='-') ) {
				alt14=1;
			}
			switch (alt14) {
				case 1 :
					// src/main/resources/org/drools/compiler/lang/DRL5Lexer.g:
					{
					if ( input.LA(1)=='+'||input.LA(1)=='-' ) {
						input.consume();
						state.failed=false;
					}
					else {
						if (state.backtracking>0) {state.failed=true; return;}
						MismatchedSetException mse = new MismatchedSetException(null,input);
						recover(mse);
						throw mse;
					}
					}
					break;

			}

			// src/main/resources/org/drools/compiler/lang/DRL5Lexer.g:95:33: ( '0' .. '9' )+
			int cnt15=0;
			loop15:
			while (true) {
				int alt15=2;
				int LA15_0 = input.LA(1);
				if ( ((LA15_0 >= '0' && LA15_0 <= '9')) ) {
					alt15=1;
				}

				switch (alt15) {
				case 1 :
					// src/main/resources/org/drools/compiler/lang/DRL5Lexer.g:
					{
					if ( (input.LA(1) >= '0' && input.LA(1) <= '9') ) {
						input.consume();
						state.failed=false;
					}
					else {
						if (state.backtracking>0) {state.failed=true; return;}
						MismatchedSetException mse = new MismatchedSetException(null,input);
						recover(mse);
						throw mse;
					}
					}
					break;

				default :
					if ( cnt15 >= 1 ) break loop15;
					if (state.backtracking>0) {state.failed=true; return;}
					EarlyExitException eee = new EarlyExitException(15, input);
					throw eee;
				}
				cnt15++;
			}

			}

		}
		finally {
			// do for sure before leaving
		}
	}
1336041
def lex(self, text):
        
        for match in self.regex.finditer(text):
            for name, _ in self.lexicon:
                m = match.group(name)
                if m is not None:
                    yield (name, m)
                    break
        yield (EOF, None)