Returns the current constant declaration key in the list of given tokens. Returns 0 if not found.

@param array $tokens the array of tokens (see http://www.php.net/manual/en/ref.tokenizer.php)
@return int
431597
private function getCurrentConstantKeyFromClassTokens($tokens)
    {
        $parsingStateConstDeclarationFound = false;
        $constDeclarationKey = 0;

        foreach ($tokens as $key => $token) {
            if (is_array($token) === true && $token[0] === T_CONST) {
                $parsingStateConstDeclarationFound = true;
                $constDeclarationKey = $key;
            }

            if ($parsingStateConstDeclarationFound === true && $token[0] === T_STRING) {
                if ($token[1] === $this->getName()) {
                    break;
                } else {
                    $parsingStateConstDeclarationFound = false;
                }
            }
        }

        return $constDeclarationKey;
    }
626676
public static function createConditionals(array $tokens) {
        if(!isset($tokens['WHERE'])) return '';

        $tableAlias = Table::alias($tokens);
        $primaryKeyColumn = sprintf('%s.%s', $tableAlias, Identifier::column($tokens, new MetaData));

        // Get WHERE conditions as string including table alias and primary key column if present
        $sqlWhereString = array_reduce($tokens['WHERE'], function($query, $token) use ($tableAlias) {
            return $query . str_replace(['"', '\''], '', str_replace('OR', '|', str_replace('AND', '&', $token['base_expr'])));
        });

        // Remove primary key column before removing table alias and returning
        return str_replace($tableAlias . '.', '', preg_replace('/' . preg_quote($primaryKeyColumn) . '=[\w\d]*&*/', '', $sqlWhereString));
    }
497784
protected function getEndIndex(array $tokens, int $index): int
    {
        $startIndex = $index;
        while (!empty($tokens[$index + 1]) && $tokens[$index + 1]['code'] !== T_DOC_COMMENT_CLOSE_TAG && $tokens[$index + 1]['code'] !== T_DOC_COMMENT_TAG) {
            $index++;
        }

        // Jump to the previous line
        $currentLine = $tokens[$index]['line'];
        while ($tokens[$index]['line'] === $currentLine) {
            $index--;
        }
        // Fix for single line doc blocks
        $index = max($index, $startIndex);

        return $this->getLastTokenOfLine($tokens, $index);
    }
90358
private function isConstant(array &$constants, array &$token, array &$previousToken)
    {
        return !empty($constants[strtoupper($token[1])]) && $previousToken[0] != \T_NAMESPACE;
    }
412098
protected function computeUsingFreqDist(array $tokens)
    {
        $freqDist = (new FreqDist($tokens))
            ->getKeyValuesByFrequency();
                
        foreach($freqDist as $token => $freqValue)
        {
            if(!isset($this->stopWords[$token])) {
                $this->stopWords[$token] = $freqValue;
            } else {
                $this->stopWords[$token] += $freqValue;
            }
        }        
    }
243824
protected function findVariablesThatReferenceClass(array $tokens) {
        $inNew     = false;
        $variables = array();

        foreach ($tokens as $i => $token) {
            if (is_string($token)) {
                if (trim($token) == ';') {
                    $inNew = false;
                }

                continue;
            }

            list ($token, $value) = $token;

            switch ($token) {
                case T_NEW:
                    $inNew = true;
                    break;

                case T_STRING:
                    if ($inNew) {
                        if ($value == $this->outClassName['fullyQualifiedClassName']) {
                            $variables[] = $this->findVariableName($tokens, $i);
                        }
                    }

                    $inNew = false;
                    break;
            }
        }

        return $variables;
    }
86410
private function parseTokens(array $tokens)
    {
        return array_map(function ($token, $key) {
            return ['name' => $key, 'value' => $token];
        }, $tokens, array_keys($tokens));
    }
284962
protected function getPrefixLength(array $strings)
	{
		$len = 1;
		$cnt = count($strings[0]);
		while ($len < $cnt && $this->stringsMatch($strings, $len))
		{
			++$len;
		}

		return $len;
	}
222832
public function parse(array $tokens) : int
    {
        $tokensAvailable = array_flip($this->getGrammar()->getTokens());

        $length = count($tokens);

        if ($length === 0) {
            throw new Exception('Invalid Roman', Exception::INVALID_ROMAN);
        }

        foreach ($tokens as $position => $token) {
            if (! is_string($token)) {
                $exception = new Exception(
                    sprintf('Invalid token type "%s" at position %d', gettype($token), $position),
                    Exception::INVALID_TOKEN_TYPE
                );

                $exception->setPosition($position);

                throw $exception;
            }

            if (! isset($tokensAvailable[$token])) {
                $exception = new Exception(
                    sprintf('Unknown token "%s" at position %d', $token, $position),
                    Exception::UNKNOWN_TOKEN
                );

                $exception
                    ->setToken($token)
                    ->setPosition($position);

                throw $exception;
            }
        }

        return (new Automaton($this->getGrammar()))->read($tokens)->getValue();
    }
383803
public function peekWhileTokens(array $tokenNames)
    {
        while ($token = $this->peek()) {
            if (!in_array($token->getName(), $tokenNames, true)) {
                break;
            }
        }

        return $token;
    }
973439
public function generateFromTokens($tokens)
    {
        if (!$tokens) {
            return '';
        }

        // Basic algorithm
        $html = '';
        for ($i = 0, $size = count($tokens); $i < $size; $i++) {
            if ($this->_scriptFix && $tokens[$i]->name === 'script'
                && $i + 2 < $size && $tokens[$i+2] instanceof HTMLPurifier_Token_End) {
                // script special case
                // the contents of the script block must be ONE token
                // for this to work.
                $html .= $this->generateFromToken($tokens[$i++]);
                $html .= $this->generateScriptFromToken($tokens[$i++]);
            }
            $html .= $this->generateFromToken($tokens[$i]);
        }

        // Tidy cleanup
        if (extension_loaded('tidy') && $this->config->get('Output.TidyFormat')) {
            $tidy = new Tidy;
            $tidy->parseString(
                $html,
                array(
                   'indent'=> true,
                   'output-xhtml' => $this->_xhtml,
                   'show-body-only' => true,
                   'indent-spaces' => 2,
                   'wrap' => 68,
                ),
                'utf8'
            );
            $tidy->cleanRepair();
            $html = (string) $tidy; // explicit cast necessary
        }

        // Normalize newlines to system defined value
        if ($this->config->get('Core.NormalizeNewlines')) {
            $nl = $this->config->get('Output.Newline');
            if ($nl === null) {
                $nl = PHP_EOL;
            }
            if ($nl !== "\n") {
                $html = str_replace("\n", $nl, $html);
            }
        }
        return $html;
    }
1145641
public static function getNamespace($tokens)
    {
        $count = count($tokens);
        $i = 0;
        $namespace = '';
        $namespace_ok = false;
        while ($i < $count) {
            $token = $tokens[$i];
            if (is_array($token) && $token[0] === T_NAMESPACE) {
                // Found namespace declaration
                while (++$i < $count) {
                    if ($tokens[$i] === ';') {
                        $namespace_ok = true;
                        $namespace = trim($namespace);
                        break;
                    }
                    $namespace .= is_array($tokens[$i]) ? $tokens[$i][1] : $tokens[$i];
                }
                break;
            }
            $i++;
        }
        if (!$namespace_ok) {
            return null;
        } else {
            return $namespace;
        }
    }
1038538
protected function getTagEndIndex(array $tokens, int $start, int $end): int
    {
        for ($i = $end; $i > $start; $i--) {
            if ($tokens[$i]['code'] !== T_DOC_COMMENT_STRING) {
                continue;
            }

            return $i;
        }

        return $start;
    }
340454
public function allCapDifferential(array $tokens) : bool
    {
        $upperCase = array_map('strtoupper', $tokens);
        $intersection = array_values( array_intersect( $tokens, $upperCase));        
        return count($intersection) > 0 && count($intersection) < count($tokens);                
    }
724379
protected function parseTypeModifiers(array $tokens)
	{
		$modifiers = 0;
	
		foreach($tokens as $tok)
		{
			if(is_array($tok))
			{
				switch($tok[0])
				{
					case T_ABSTRACT:
						$modifiers |= MetaInfo::IS_ABSTRACT;
						break;
					case T_FINAL:
						$modifiers |= MetaInfo::IS_FINAL;
						break;
				}
			}
		}
	
		return $modifiers;
	}
161809
public static function create(array $tokens) {
        HashMap::assert($tokens, 'tokens');

        if (empty($tokens['LIMIT'])) return [];

        $headers = [];
        array_push($headers, 'Limit: ' . $tokens['LIMIT']['rowcount']);
        if(empty($tokens['LIMIT']['offset'])) return $headers;
        array_push($headers, 'Offset: ' . $tokens['LIMIT']['offset']);
        return $headers;
    }
280104
public function nextOf($tokenNames)
    {
        $i = $this->index+1;
        $len = count($this->tokens);
        while ($i < $len) {
            $tok = $this->tokens[$i][0];
            if (in_array($tok, $tokenNames)) {
                return $tok;
            }
            $i++;
        }
        return null;
    }