{@inheritdoc}

@throws LexException
962115
public function lex(string $value): array
    {
        foreach (self::PATTERNS as $pattern => $tokenTypeConstant) {
            if (1 === preg_match($pattern, $value, $matches)) {
                if (null === $tokenTypeConstant) {
                    throw InvalidArgumentExceptionFactory::createForInvalidExpressionLanguageToken($value);
                }

                return [new Token($matches[0], new TokenType($tokenTypeConstant))];
            }
        }

        return $this->lexer->lex($value);
    }
1234716
protected function lexInterfaceTypeExtension(): InterfaceTypeExtensionNode
    {
        $start = $this->lexer->getToken();

        $this->expectKeyword(KeywordEnum::EXTEND);
        $this->expectKeyword(KeywordEnum::INTERFACE);

        $name       = $this->lexName();
        $directives = $this->lexDirectives();
        $fields     = $this->lexFieldsDefinition();

        if (empty($directives) && empty($fields)) {
            throw $this->unexpected();
        }

        return new InterfaceTypeExtensionNode($name, $directives, $fields, $this->createLocation($start));
    }
1140443
public function lexTpl()
    {
        $source = $this->source;
        if ($source->cursor >= $source->bound) {
            $this->addError("The parser did not find the '{$source->leftDelimiter}' template start tag", $source->cursor);
            return null;
        }
        Rules::reset($source->leftDelimiter, $source->rightDelimiter);
        $this->stack = [];
        $tree = new TreeMap();
        if (Sdopx::$debug) {
            $tree->setDebugInfo($source->getDebugInfo());
        }
        $next = ['openTpl' => 1];
        do {
            $data = $this->match($next);

            if ($data == null) {
                $this->addError("Template tag syntax is incorrect", $source->cursor);
                return null;
            }
            $tag = $data['tag'];
            $source->cursor = $data['end'];
            //清除
            $item = Rules::getItem($tag);
            if (isset($item['clear'])) {
                $clear = $item['clear'];
                $flag = end($this->stack);
                while ($flag !== null && ($flag & $clear) > 0) {
                    array_pop($this->stack);
                    $flag = end($this->stack);
                }
            }
            //出栈
            if (isset($item['close'])) {
                $close = $item['close'];
                $flag = end($this->stack);
                if ($flag !== null) {
                    if (($flag & $close) == 0) {
                        $this->addError("Template tag syntax parsing error, unclosed syntax.", $source->cursor);
                        return null;
                    }
                    array_pop($this->stack);
                }
            }
            //入栈
            if (isset($item['open'])) {
                array_push($this->stack, $item['open']);
            }
            if ($tag == 'closeTpl') {
                if (count($this->stack) != 0) {
                    $this->addError("Template tag syntax parsing error, as well as unclosed syntax tags.", $source->cursor);
                    return null;
                }
                $data['node'] = null;
                $tree->push($data);
                return $tree;
            }
            $data['node'] = end($this->stack);
            $tree->push($data);
            $next = Rules::getItem($tag, 'next');
        } while (true);
    }
817697
private function handleChildBlock(Lexer $lexer, int $useragentPosition, int $childPosition) : int
    {
        $enterPlatforms = false;
        $enterDevices   = false;
        $collectMatch   = false;

        $functionStart       = $this->getLocationCoordinates($lexer);
        $functionDeclaration = [];

        do {
            $code = $lexer->lex();

            switch ($code) {
                case self::JSON_STRING:
                    if ('platforms' === $lexer->yytext) {
                        $enterPlatforms = true;
                    } elseif ('devices' === $lexer->yytext) {
                        $enterDevices = true;
                    } elseif ('match' === $lexer->yytext) {
                        $collectMatch = true;
                    } elseif (true === $collectMatch) {
                        $collectMatch        = false;
                        $match               = $lexer->yytext;
                        $functionDeclaration = [
                            'start' => $this->getLocationCoordinates($lexer, false, '"' . $match . '"'),
                            'end' => $this->getLocationCoordinates($lexer, true, '"' . $match . '"'),
                        ];
                    }

                    break;
                case self::JSON_OBJECT_START:
                    if (true === $enterDevices) {
                        $code         = $this->handleDeviceBlock($lexer, $useragentPosition, $childPosition);
                        $enterDevices = false;
                    } else {
                        $code = $this->ignoreObjectBlock($lexer);
                    }

                    break;
                case self::JSON_ARRAY_START:
                    if (true === $enterPlatforms) {
                        $code           = $this->handlePlatformBlock($lexer, $useragentPosition, $childPosition);
                        $enterPlatforms = false;
                    }

                    break;
            }
        } while (self::JSON_OBJECT_END !== $code);

        $functionEnd = $this->getLocationCoordinates($lexer, true);

        $functionCoverage = $this->getCoverageCount(
            sprintf('u%d::c%d::d::p', $useragentPosition, $childPosition),
            $this->fileCoveredIds
        );

        $this->collectFunction($functionStart, $functionEnd, $functionDeclaration, $functionCoverage);

        return $lexer->lex();
    }
843926
private void throwMessage(int number, String msg) throws LexException
	{
		throwMessage(number, linecount, charpos, msg);
	}
315002
public function prepare(Lexer $lexer) {
        $this->acceptedRuleNames = $this->set->getRuleNamesByCategory(AbstractRuleSet::CATEGORY_FORMAT);
    }
604151
def lex(self, text):
        """"""
        for match in self.regex.finditer(text):
            name = match.lastgroup
            yield (name, match.group(name))
1336041
def lex(self, text):
        
        for match in self.regex.finditer(text):
            for name, _ in self.lexicon:
                m = match.group(name)
                if m is not None:
                    yield (name, m)
                    break
        yield (EOF, None)
1170033
def _lex(stream):
    
    lines = enumerate(stream, 1)
    line_no = pos = 0
    try:
        while True:
            if pos == 0:
                line_no, line = next(lines)
            matches = _tdl_lex_re.finditer(line, pos)
            pos = 0  # reset; only used for multiline patterns
            for m in matches:
                gid = m.lastindex
                if gid <= 2:  # potentially multiline patterns
                    if gid == 1:  # docstring
                        s, start_line_no, line_no, line, pos = _bounded(
                            '"""', '"""', line, m.end(), line_no, lines)
                    elif gid == 2:  # comment
                        s, start_line_no, line_no, line, pos = _bounded(
                            '#|', '|#', line, m.end(), line_no, lines)
                    yield (gid, s, line_no)
                    break
                elif gid == 30:
                    raise TdlParsingError(
                        ('Syntax error:\n  {}\n {}^'
                         .format(line, ' ' * m.start())),
                        line_number=line_no)
                else:
                    # token = None
                    # if not (6 < gid < 20):
                    #     token = m.group(gid)
                    token = m.group(gid)
                    yield (gid, token, line_no)
    except StopIteration:
        pass
938743
protected function lexExecutableDefinition(): ExecutableDefinitionNodeInterface
    {
        if ($this->peek(TokenKindEnum::NAME)) {
            // Valid names are: query, mutation, subscription and fragment
            $token = $this->lexer->getToken();

            switch ($token->getValue()) {
                case KeywordEnum::QUERY:
                case KeywordEnum::MUTATION:
                case KeywordEnum::SUBSCRIPTION:
                    return $this->lexOperationDefinition();
                case KeywordEnum::FRAGMENT:
                    return $this->lexFragmentDefinition();
            }
        } elseif ($this->peek(TokenKindEnum::BRACE_L)) {
            // Anonymous query
            return $this->lexOperationDefinition();
        }

        throw $this->unexpected();
    }
251756
public function lex(string $value): array
    {
        $escapedValue = preg_replace('/(\\p{L})@/', '$1\\@', $value);

        return $this->lexer->lex($escapedValue);
    }
1329124
@Requires("nextToken == null")
  @Ensures("result == (nextToken != null)")
  protected boolean lex() throws IOException {
    int startOffset = currentOffset;
    StringBuilder buffer = new StringBuilder();
    int c = readChar(true);
    if (c == -1) {
      return false;
    }
    buffer.append((char) c);
    switch (c) {
      case '/':
        /* Comments. */
        c = readChar(true);
        switch (c) {
          case '/':
            buffer.append((char) c);
            do {
              c = readChar(false);
              buffer.append((char) c);
            } while (c != '\n');
            nextToken = new Token(TokenKind.COMMENT, buffer.toString(),
                                  startOffset);
            break;
          case '*':
            buffer.append((char) c);
            for (;;) {
              c = readChar(false);
              buffer.append((char) c);
              if (c == '*') {
                c = readChar(false);
                buffer.append((char) c);
                if (c == '/') {
                  break;
                }
              }
            }
            nextToken = new Token(TokenKind.COMMENT, buffer.toString(),
                                  startOffset);
            break;
          default:
            unreadChar(c);
            nextToken = new Token(TokenKind.SYMBOL, buffer.toString(),
                                  startOffset);
        }
        break;

      case '\'':
      case '"':
        /* Quoted strings. */
        int delim = c;
        for (;;) {
          c = readChar(false);
          buffer.append((char) c);
          if (c == delim) {
            break;
          }
          if (c == '\\') {
            buffer.append((char) readChar(false));
          }
        }
        nextToken = new Token(TokenKind.QUOTE, buffer.toString(), startOffset);
        break;

      default:
        if (Character.isJavaIdentifierStart(c)) {
          /* Identifiers. */
          while ((c = readChar(true)) != -1
                 && Character.isJavaIdentifierPart(c)) {
            buffer.append((char) c);
          }
          unreadChar(c);
          nextToken = new Token(TokenKind.WORD, buffer.toString(), startOffset);
        } else if (Character.isWhitespace(c)) {
          /* White space. */
          while ((c = readChar(true)) != -1 && Character.isWhitespace(c)) {
            buffer.append((char) c);
          }
          unreadChar(c);
          nextToken = new Token(TokenKind.SPACE, buffer.toString(),
                                startOffset);
        } else {
          /* Symbol. */
          nextToken = new Token(TokenKind.SYMBOL, buffer.toString(),
                                startOffset);
        }
    }
    return true;
  }
422950
function lexicon($path) {
		$lex=array();
		foreach ($this->languages?:explode(',',$this->fallback) as $lang)
			foreach ($this->split($path) as $dir)
				if ((is_file($file=($base=$dir.$lang).'.php') ||
					is_file($file=$base.'.php')) &&
					is_array($dict=require($file)))
					$lex+=$dict;
				elseif (is_file($file=$base.'.ini')) {
					preg_match_all(
						'/(?<=^|\n)(?:'.
							'\[(?<prefix>.+?)\]|'.
							'(?<lval>[^\h\r\n;].*?)\h*=\h*'.
							'(?<rval>(?:\\\\\h*\r?\n|.+?)*)'.
						')(?=\r?\n|$)/',
						$this->read($file),$matches,PREG_SET_ORDER);
					if ($matches) {
						$prefix='';
						foreach ($matches as $match)
							if ($match['prefix'])
								$prefix=$match['prefix'].'.';
							elseif (!array_key_exists(
								$key=$prefix.$match['lval'],$lex))
								$lex[$key]=trim(preg_replace(
									'/\\\\\h*\r?\n/','',$match['rval']));
					}
				}
		return $lex;
	}
1202631
private function match($token)
    {
        if (!$this->lexer->isNextToken($token)) {
            throw SyntaxException::expectedToken(
                $this->lexer->getLiteral($token),
                null,
                $this->lexer
            );
        }

        return $this->lexer->moveNext();
    }
1287100
protected function lexFragment(): FragmentNodeInterface
    {
        $start = $this->lexer->getToken();

        $this->expect(TokenKindEnum::SPREAD);

        $token = $this->lexer->getToken();

        if (KeywordEnum::ON !== $token->getValue() && $this->peek(TokenKindEnum::NAME)) {
            return new FragmentSpreadNode(
                $this->lexFragmentName($token),
                $this->lexDirectives(),
                null,
                $this->createLocation($start)
            );
        }

        if (KeywordEnum::ON === $token->getValue()) {
            $this->lexer->advance();
            $typeCondition = $this->lexNamedType();
        }

        return new InlineFragmentNode(
            $typeCondition ?? null,
            $this->lexDirectives(),
            $this->lexSelectionSet(),
            $this->createLocation($start)
        );
    }
602324
public function lex(string $value): array
    {
        foreach (self::PATTERNS as $pattern => $tokenTypeConstant) {
            if (1 === preg_match($pattern, $value, $matches)) {
                if (null === $tokenTypeConstant) {
                    throw InvalidArgumentExceptionFactory::createForInvalidExpressionLanguageToken($value);
                }

                return [new Token($matches[0], new TokenType($tokenTypeConstant))];
            }
        }

        throw ExpressionLanguageExceptionFactory::createForCouldNotLexValue($value);
    }
804362
public function prepare(Lexer $lexer) {
        $this->openingTokens = $this->additionalOpeningTokens;
        $this->closingTokens = $this->additionalClosingTokens;

        $singleRuleNames = $this->set->getRuleNamesByCategory(MarkdownRuleSet::CATEGORY_SINGLE);
        foreach ($singleRuleNames as $singleRuleName) {
            $this->closingTokens[] = $singleRuleName;
        }

        $blockRuleNames = $this->set->getRuleNamesByCategory(MarkdownRuleSet::CATEGORY_BLOCK);
        foreach ($blockRuleNames as $blockRuleName) {
            $this->openingTokens[] = $blockRuleName . AbstractBlockRule::CLOSE;
            $this->closingTokens[] = $blockRuleName . AbstractBlockRule::OPEN;
            $this->closingTokens[] = $blockRuleName . AbstractBlockRule::CLOSE;
        }

        return $this;
    }