@param Tokenizer  $tokenizer
@param string     $string
@param array|null $previous

@return array
334811
public static function isReserved(Tokenizer $tokenizer, $string, $previous)
    {
        $tokenData = [];

        if (!$tokenizer->getNextToken() && self::isReservedPrecededByDotCharacter($previous)) {
            $upperCase = \strtoupper($string);

            self::getReservedString($tokenData, Tokenizer::TOKEN_TYPE_RESERVED_TOP_LEVEL, $string, $tokenizer);
            self::getReservedString($tokenData, Tokenizer::TOKEN_TYPE_RESERVED_NEWLINE, $upperCase, $tokenizer);
            self::getReservedString($tokenData, Tokenizer::TOKEN_TYPE_RESERVED, $string, $tokenizer);

            $tokenizer->setNextToken($tokenData);
        }
    }
852672
protected static function parseExpression(TokenIterator $tokens, \ArrayIterator $options) {
        $seq = static::parseSequence($tokens, $options);
        if ($tokens->current() != '|')
            return $seq;

        $result = null;
        if (sizeof($seq) > 1) $result = [new Required($seq)];
        else                  $result = $seq;

        while ($tokens->current() == '|') {
            $tokens->move();
            $seq = static::parseSequence($tokens, $options);
            if (sizeof($seq) > 1) $result[] = new Required($seq);
            else                  $result   = array_merge($result, $seq);
        }

        if (sizeof($result) > 1)
            return new Either($result);
        return $result;
    }
825768
private TextToken text(Token prev) {
    int startIndex = curCharIndex;

    while (curChar != '{' && curChar != EOF) {
      consume();
    }
    return new TextToken(prev, curCharIndex - startIndex);
  }
1308814
public function tokenize($string)
    {
        if(!$this->length) {
            return array(substr($string, $this->startPosition));
        } else {
            return array(substr($string, $this->startPosition, $this->length));
        }
    }
1225867
public function tokenize($string)
    {
        // Get the list of sorted operators.
        $operators = $this->factory->getOperators();

        // Get the operator regular expression.
        $regex = $this->getRegex($operators);

        // Check that we have matched all the tokens in the string.
        if (!preg_match_all($regex, $string, $matches, PREG_SET_ORDER)) {
            throw new LexerException('Unable to tokenize string.');
        }

        // Cycle through all available tokens.
        $tokens = [];
        foreach ($matches as $match) {
            $item = $match[0];

            // Process the parentheses as special cases.
            if (in_array($item, ['(', ')', ','])) {
                $tokens[] = new Token($item);
                continue;
            }

            // Locate the first operator that matches the token.
            /** @var \Xylemical\Expressions\Operator $operator */
            foreach ($operators as $operator) {
                if (preg_match('#^' . $operator->getRegex() . '$#i', $item)) {
                    $tokens[] = new Token($item, $operator);
                    break;
                }
            }
        }

        return $tokens;
    }
1072066
function getTokens (string) {
  let tokens = null
  try { tokens = esprima.tokenize('`'+ string +'`') }
  catch (e) { tokens = [] }
  return tokens
}
1221795
private function extractUseStatements(\ArrayIterator $tokens)
    {
        $extractedUseStatements = [];
        $groupedNs = '';
        $currentNs = '';
        $currentAlias = null;
        $state = 'start';

        $i = 0;
        while ($tokens->valid()) {
            $i += 1;
            $currentToken = $tokens->current();
            $tokenId = is_string($currentToken) ? $currentToken : $currentToken[0];
            $tokenValue = is_string($currentToken) ? null : $currentToken[1];
            switch ($state) {
                case 'start':
                    switch ($tokenId) {
                        case T_STRING:
                        case T_NS_SEPARATOR:
                            $currentNs .= $tokenValue;
                            break;
                        case T_CURLY_OPEN:
                        case '{':
                            $state = 'grouped';
                            $groupedNs = $currentNs;
                            break;
                        case T_AS:
                            $state = 'start-alias';
                            break;
                        case self::T_LITERAL_USE_SEPARATOR:
                        case self::T_LITERAL_END_OF_USE:
                            $state = 'end';
                            break;
                        default:
                            break;
                    }
                    break;
                case 'start-alias':
                    switch ($tokenId) {
                        case T_STRING:
                            $currentAlias .= $tokenValue;
                            break;
                        case self::T_LITERAL_USE_SEPARATOR:
                        case self::T_LITERAL_END_OF_USE:
                            $state = 'end';
                            break;
                        default:
                            break;
                    }
                    break;
                case 'grouped':
                    switch ($tokenId) {
                        case T_STRING:
                        case T_NS_SEPARATOR:
                            $currentNs .= $tokenValue;
                            break;
                        case T_AS:
                            $state = 'grouped-alias';
                            break;
                        case self::T_LITERAL_USE_SEPARATOR:
                            $state = 'grouped';
                            $extractedUseStatements[$currentAlias ?: $currentNs] = $currentNs;
                            $currentNs = $groupedNs;
                            $currentAlias = null;
                            break;
                        case self::T_LITERAL_END_OF_USE:
                            $state = 'end';
                            break;
                        default:
                            break;
                    }
                    break;
                case 'grouped-alias':
                    switch ($tokenId) {
                        case T_STRING:
                            $currentAlias .= $tokenValue;
                            break;
                        case self::T_LITERAL_USE_SEPARATOR:
                            $state = 'grouped';
                            $extractedUseStatements[$currentAlias ?: $currentNs] = $currentNs;
                            $currentNs = $groupedNs;
                            $currentAlias = null;
                            break;
                        case self::T_LITERAL_END_OF_USE:
                            $state = 'end';
                            break;
                        default:
                            break;
                    }
            }

            if ($state === 'end') {
                break;
            }

            $tokens->next();
        }

        if ($groupedNs !== $currentNs) {
            $extractedUseStatements[$currentAlias ?: $currentNs] = $currentNs;
        }

        return $extractedUseStatements;
    }
1025
protected function getTokenStrings(int $row, array $tokens): array
    {
        $pieces = [];
        foreach ($tokens as $key => $token) {
            if ($token['line'] > $row) {
                break;
            }
            if ($token['line'] < $row) {
                continue;
            }
            if ($this->verbose) {
                $type = $token['type'];
                $content = $token['content'];
                $content = '`' . str_replace(["\r\n", "\n", "\r", "\t"], ['\r\n', '\n', '\r', '\t'], $content) . '`';

                unset($token['type']);
                unset($token['content']);
                $token['content'] = $content;

                $tokenList = [];
                foreach ($token as $k => $v) {
                    if (is_array($v)) {
                        if (empty($v)) {
                            continue;
                        }
                        $v = json_encode($v);
                    }
                    $tokenList[] = $k . '=' . $v;
                }
                $pieces[] = $type . ' (' . $key . ') ' . implode(', ', $tokenList);
            } else {
                $pieces[] = $token['type'];
            }
        }
        if ($this->verbose) {
            return $pieces;
        }
        return [implode(' ', $pieces)];
    }
898852
private function extractUseStatement(\ArrayIterator $tokens)
    {
        $result = [''];
        while ($tokens->valid()
            && (self::T_LITERAL_USE_SEPARATOR !== $tokens->current()[0])
            && (self::T_LITERAL_END_OF_USE !== $tokens->current()[0])
        ) {
            if (T_AS === $tokens->current()[0]) {
                $result[] = '';
            }
            if (T_STRING === $tokens->current()[0] || T_NS_SEPARATOR === $tokens->current()[0]) {
                $result[\count($result) - 1] .= $tokens->current()[1];
            }
            $tokens->next();
        }

        if (1 == \count($result)) {
            $backslashPos = strrpos($result[0], '\\');

            if (false !== $backslashPos) {
                $result[] = substr($result[0], $backslashPos + 1);
            } else {
                $result[] = $result[0];
            }
        }

        return array_reverse($result);
    }
733319
static function Tokenize($string, $splitChars = ' %$§&/()?!.:;\#\'"')
    {
       $tokens = array();

       $tok = strtok($string,  $splitChars);
       while ($tok !== false)
       {
            array_push($tokens, $tok);
            $tok = strtok($splitChars);
       }
       return $tokens;
    }
1055117
public static MutableList<String> tokensToList(String string, String separator)
    {
        MutableList<String> list = Lists.mutable.empty();
        for (StringTokenizer stringTokenizer = new StringTokenizer(string, separator); stringTokenizer.hasMoreTokens(); )
        {
            String token = stringTokenizer.nextToken();
            list.add(token);
        }
        return list;
    }
779032
protected function getNextTokenFromString($string, $token, $cacheKey)
    {
        $token = $this->parseNextToken($string, $token);

        if ($cacheKey && \strlen($token[self::TOKEN_VALUE]) < $this->maxCacheKeySize) {
            $this->tokenCache[$cacheKey] = $token;
        }

        return $token;
    }
389021
public static String[] tokenize(String str, String delimiters) {
        return tokenize(str, delimiters, false);
    }
4200
protected static function isReservedPrecededByDotCharacter($previous)
    {
        return !$previous || !isset($previous[Tokenizer::TOKEN_VALUE]) || $previous[Tokenizer::TOKEN_VALUE] !== '.';
    }
418554
public function parseToken(string $tokenString)
    {
        // try to parse a token string.
        try {
            return $this->parser()->parse($tokenString);
        } catch (\Exception $e) {
            // if it was not possible to, return null.
            return null;
        }
    }
37096
public static function parse(Tokenizer $tokenizer, StreamReader $streamReader, PdfParser $parser)
    {
        $entries = [];

        while (true) {
            $token = $tokenizer->getNextToken();
            if ($token === '>' && $streamReader->getByte() === '>') {
                $streamReader->addOffset(1);
                break;
            }

            $key = $parser->readValue($token);
            if ($key === false) {
                return false;
            }

            // ensure the first value to be a Name object
            if (!($key instanceof PdfName)) {
                $lastToken = null;
                // ignore all other entries and search for the closing brackets
                while (($token = $tokenizer->getNextToken()) !== '>' && $token !== false && $lastToken !== '>') {
                    $lastToken = $token;
                }

                if ($token === false) {
                    return false;
                }

                break;
            }


            $value = $parser->readValue();
            if ($value === false) {
                return false;
            }

            if ($value instanceof PdfNull) {
                continue;
            }

            // catch missing value
            if ($value instanceof PdfToken && $value->value === '>' && $streamReader->getByte() === '>') {
                $streamReader->addOffset(1);
                break;
            }

            $entries[$key->value] = $value;
        }

        $v = new self;
        $v->value = $entries;

        return $v;
    }
830082
private function tokenize($string, $separator = '')
    {
        if (!strcmp($separator, '')) {
            $separator = $string;
            $string = $this->headerNextToken;
        }

        for ($character = 0; $character < strlen($separator); $character++) {
            if (gettype($position = strpos($string, $separator[$character])) == 'integer') {
                $found = (isset($found) ? min($found, $position) : $position);
            }
        }

        if (isset($found)) {
            $this->headerNextToken = substr($string, $found + 1);

            return substr($string, 0, $found);
        } else {
            $this->headerNextToken = '';

            return $string;
        }
    }