@param Token[] $tokens
@param string  $tokenClass FQCN of the token class to filter for

@return Token[]
544098
private function filterTokensByClass(array $tokens, $tokenClass)
    {
        return array_filter(
            $tokens,
            function (Token $token) use ($tokenClass) {
                return get_class($token) === $tokenClass;
            }
        );
    }
908437
private function getLicenseTokenIndex(array $tokens)
    {
        $licenseTokenIndex = null;

        foreach ($tokens as $index => $token) {
            if ($token[0] === T_COMMENT) {
                $licenseTokenIndex = $index;
            }

            // if we reach the class declaration then it does not have a license
            if ($token[0] === T_CLASS) {
                break;
            }
        }

        return $licenseTokenIndex;
    }
207823
private function getClassNamesFromTokens(array $tokens) : array
    {
        for ($i = 0;$i < count($tokens);$i++) {
            // Skip literals
            if (is_string($tokens[$i])) {
                continue;
            }

            $className = '';

            switch ($tokens[$i][0]) {
                case T_NAMESPACE:
                    $namespace = '';

                    // Collect all the namespace parts and separators
                    while (isset($tokens[++$i][1])) {
                        if (in_array($tokens[$i][0], [T_NS_SEPARATOR, T_STRING])) {
                            $namespace .= $tokens[$i][1];
                        }
                    }

                    break;
                case T_CLASS:
                    $isClassConstant = false;

                    // Scan previous tokens to see if they're double colons, which would mean this is a class constant
                    for ($j = $i - 1;$j >= 0;$j--) {
                        if (!isset($tokens[$j][1])) {
                            break;
                        }

                        if ($tokens[$j][0] === T_DOUBLE_COLON) {
                            $isClassConstant = true;
                            break 2;
                        } elseif ($tokens[$j][0] === T_WHITESPACE) {
                            // Since we found whitespace, then we know this isn't a class constant
                            break;
                        }
                    }

                    // Get the class name
                    while (isset($tokens[++$i][1])) {
                        if ($tokens[$i][0] === T_STRING) {
                            $className .= $tokens[$i][1];
                            break;
                        }
                    }

                    $classNames[] = ltrim($namespace . '\\' . $className, '\\');
                    break 2;
            }
        }

        return $classNames;
    }
503731
public function parse(array $tokens)
    {
        $parser = new Parser($this->symbolContainer);

        if ($this->customGrammarChecker !== null) {
            $parser->setCustomGrammarChecker($this->customGrammarChecker);
        }

        $rootNode = $parser->parse($tokens);

        return $rootNode;
    }
329206
public function getTokenModel($modelClass = 'StandardTokenModel')
    {
        if (! isset($this->_tokenModels[$modelClass])) {
            $this->_tokenModels[$modelClass] = $this->_loadClass('Model_' . $modelClass, true);
            $this->_tokenModels[$modelClass]->applyFormatting();
        }

        return $this->_tokenModels[$modelClass];
    }
603956
private function matchUseImports(Tokens $tokens, string $className): ?string
    {
        $namespaceUseAnalyses = $this->namespaceUsesAnalyzer->getDeclarationsFromTokens($tokens);

        foreach ($namespaceUseAnalyses as $namespaceUseAnalysis) {
            if ($className === $namespaceUseAnalysis->getShortName()) {
                return $namespaceUseAnalysis->getFullName();
            }
        }

        return null;
    }
996650
public static function alias(array $tokens) {
        HashMap::assert($tokens, 'tokens');

        $operation = SqlOperation::create($tokens);
        if ($operation === SqlOperations::INSERT) return null;
        if ($operation === SqlOperations::UPDATE) return $tokens['UPDATE'][0]['alias']['name'];
        return $tokens['FROM'][0]['alias']['name'];
    }
412098
protected function computeUsingFreqDist(array $tokens)
    {
        $freqDist = (new FreqDist($tokens))
            ->getKeyValuesByFrequency();
                
        foreach($freqDist as $token => $freqValue)
        {
            if(!isset($this->stopWords[$token])) {
                $this->stopWords[$token] = $freqValue;
            } else {
                $this->stopWords[$token] += $freqValue;
            }
        }        
    }
586890
protected static function parseAtom(TokenIterator $tokens, \ArrayIterator $options) {
        $tokenError = $tokens->getTokenError();
        $token      = $tokens->current();
        $result     = [];

        if ($token=='(' || $token=='[') {
            $tokens->move();

            static $index; !$index && $index = [
                '(' => [')', Required::class],
                '[' => [']', Optional::class],
            ];
            list ($matching, $patternClass) = $index[$token];

            $result = new $patternClass(static::parseExpression($tokens, $options));
            if ($tokens->move() != $matching)
                throw new $tokenError('Unmatched "'.$token.'"');
            return [$result];
        }
        elseif ($token == 'options') {
            $tokens->move();
            return [new OptionsShortcut()];
        }
        elseif (strpos($token, '--')===0 && $token!='--') {
            return static::parseLong($tokens, $options);
        }
        elseif (strpos($token, '-')===0 && $token!='-' && $token!='--') {
            return static::parseShort($tokens, $options);
        }
        elseif ((strpos($token, '<')===0 && strEndsWith($token, '>')) || static::isUpperCase($token)) {
            return [new Argument($tokens->move())];
        }
        else {
            return [new Command($tokens->move())];
        }
    }
435193
public static function modifiers($tokenId, array $tokens) {
		if (!in_array($tokens[$tokenId]['id'], array(T_CLASS, T_FUNCTION, T_VARIABLE))) {
			$token = print_r($tokens[$tokenId], true);
			throw new \Exception('Cannot call modifiers on non class/function/variable' . $token);
		}
		$modifiers = array();
		for ($key = $tokenId - 1; $key >= 0; $key--) {
			$token = $tokens[$key];
			if ($token['id'] === T_WHITESPACE) {
				continue;
			} elseif (in_array($token['id'], static::$modifiers)) {
				$modifiers[] = $key;
			} else {
				break;
			}
		}
		return $modifiers;
	}
1371055
private function getFields($tokens, $namingConvention, $class)
    {
        $fields = [];
        $prefix = '';

        foreach ($tokens->getTokens() as $token) {
            if ($token instanceof ByToken) {
                return $prefix . $this->fieldsToSQL($fields, $class, $namingConvention);
            }

            if ($token instanceof AllToken) {
                return $prefix . $this->fieldsToSQL([], $class, $namingConvention);
            }

            if($token instanceof DistinctToken) {
                $prefix .= 'DISTINCT ';
                continue;
            }

            if ($token instanceof AndToken) {
                // Ignore and tokens as they do nothing
                continue;
            }

            if (!($token instanceof ValueToken)) {
                throw new DslSyntaxException('Unexpected token ' . $token->getName() . ' in field selector', DATADO_UNEXPECTED_TOKEN);
            }

            $fields[] = lcfirst($token->getSource());
        }

        return $prefix . $this->fieldsToSQL($fields, $class, $namingConvention);
    }
594549
protected function getNamedCapturesExpressions(array $tokens)
	{
		$exprs = [];
		foreach ($tokens as $token)
		{
			if ($token['type'] !== 'capturingSubpatternStart' || !isset($token['name']))
			{
				continue;
			}

			$expr = $token['content'];
			if (strpos($expr, '|') !== false)
			{
				$expr = '(?:' . $expr . ')';
			}

			$exprs[$token['name']] = $expr;
		}

		return $exprs;
	}
1202932
private function sortTokens(array $tokens): array
    {
        $tagCounts = $this->docTagHelper->getTagCounts($tokens);

        usort($tokens, function (array $leftToken, array $rightToken) use ($tagCounts): int {
            return $this->compareTokensForSorting($leftToken, $rightToken, $tagCounts);
        });

        return $tokens;
    }
952102
def tokens(cls, tokens):
        
        return cls(Lnk.TOKENS, tuple(map(int, tokens)))
233203
protected function detectSymbols(array $tokens)
    {
        $symbolNodes = [];

        $expectingOpeningBracket = false; // True if we expect an opening bracket (after a function name)
        $openBracketCounter = 0;

        foreach ($tokens as $token) {
            $type = $token->getType();

            if ($type == Token::TYPE_WORD) {
                $identifier = $token->getValue();
                $symbol = $this->symbolContainer->find($identifier);

                if ($symbol === null) {
                    $this->throwException(
                        NotFoundException::class,
                        'Error: Detected unknown or invalid string identifier.',
                        $token->getPosition(),
                        $identifier
                    );
                }
            } elseif ($type == Token::TYPE_NUMBER) {
                // Notice: Numbers do not have an identifier
                $symbol = $this->symbolContainer->findSubtypes(Number::class)[0];
            } else { // Type Token::TYPE_CHARACTER:
                $identifier = $token->getValue();
                $symbol = $this->symbolContainer->find($identifier);

                if ($symbol === null) {
                    $this->throwException(
                        NotFoundException::class,
                        'Error: Detected unknown or invalid character identifier.',
                        $token->getPosition(),
                        $identifier
                    );
                }

                if (is_a($symbol, AbstractOpeningBracket::class)) {
                    $openBracketCounter++;
                }
                if (is_a($symbol, AbstractClosingBracket::class)) {
                    $openBracketCounter--;

                    // Make sure there are not too many closing brackets
                    if ($openBracketCounter < 0) {
                        $this->throwException(
                            ParserException::class,
                            'Error: Found closing bracket that does not have an opening bracket.',
                            $token->getPosition(),
                            $identifier
                        );
                    }
                }
            }

            // Make sure a function is not followed by a symbol that is not of type opening bracket
            if ($expectingOpeningBracket) {
                if (! is_a($symbol, AbstractOpeningBracket::class)) {
                    $this->throwException(
                        ParserException::class,
                        'Error: Expected opening bracket (after a function) but got something else.',
                        $token->getPosition(),
                        get_class($symbol)
                    );
                }

                $expectingOpeningBracket = false;
            } else {
                if (is_a($symbol, AbstractFunction::class)) {
                    $expectingOpeningBracket = true;
                }
            }

            $symbolNode = new SymbolNode($token, $symbol);

            $symbolNodes[] = $symbolNode;
        }

        // Make sure the term does not end with the name of a function but without an opening bracket
        if ($expectingOpeningBracket) {
            throw new ParserException(
                'Error: Expected opening bracket (after a function) but reached the end of the term'
            );
        }

        // Make sure there are not too many opening brackets
        if ($openBracketCounter > 0) {
            throw new ParserException(
                'Error: There is at least one opening bracket that does not have a closing bracket'
            );
        }

        return $symbolNodes;
    }
394907
public function parse(array $tokens = array())
    {
        $this->lineNum    = -1;
        $this->lineTokens = 0;
        $this->pragmas    = $this->defaultPragmas;

        $this->pragmaFilters = isset($this->pragmas[Mustache_Engine::PRAGMA_FILTERS]);
        $this->pragmaBlocks  = isset($this->pragmas[Mustache_Engine::PRAGMA_BLOCKS]);

        return $this->buildTree($tokens);
    }
1284075
public static function printTokens(
        InputInterface $input,
        OutputInterface $output,
        array $tokens
    ) {
        $withMetadata = $input->getOption('with-metadata');
        $table = new Table($output);
        $headers = ['UUID', 'Indices', 'endpoints', 'plugins', 'ttl'];
        if ($withMetadata) {
            $headers[] = 'Metadata';
        }
        $table->setHeaders($headers);

        foreach ($tokens as $token) {
            $row = [
                $token->getTokenUUID()->composeUUID(),
                implode(', ', array_map(function (IndexUUID $index) {
                    return $index->composeUUID();
                }, $token->getIndices())),
                implode(', ', $token->getEndpoints()),
                implode(', ', $token->getPlugins()),
                $token->getTtl(),
            ];

            if ($withMetadata) {
                $metadataRow = [];
                foreach ($token->getMetadata() as $metadataField => $metadataValue) {
                    $metadataRow[] = sprintf("$metadataField = %s",
                        is_bool($metadataValue)
                            ? $metadataValue ? 'true' : 'false'
                            : (
                                is_array($metadataValue)
                                    ? implode(', ', $metadataValue)
                                    : (string) $metadataValue
                            )
                    );
                }
                $row[] = implode(PHP_EOL, $metadataRow);
            }

            $table->addRow($row);
        }

        $table->render();
    }