Initialize the parser with given array of $tokens.

@param \QueryTranslator\Values\Token[] $tokens
338885
private function init(array $tokens)
    {
        $this->corrections = [];
        $this->tokens = $tokens;
        $this->cleanupGroupDelimiters($this->tokens);
        $this->stack = new SplStack();
    }
672553
protected function parseTokensContent(array $tokens): string
    {
        $statement = '';
        foreach ($tokens as $token) {
            $statement .= $token['content'];
        }

        return $statement;
    }
182760
public function registerParsers(array $parsers)
	{
		foreach ($parsers as $extension => $parser)
		{
			$this->registerParser($extension, $parser);
		}
	}
958891
public function translate(array $tokens)
    {
        $this->operatorStack = new SplStack();
        $this->outputQueue = new SplQueue();
        foreach($tokens as $token) {
            switch ($token->getType()) {
                case Token::T_OPERAND:
                    $this->outputQueue->enqueue($token);
                    break;
                case Token::T_OPERATOR:
                    $o1 = $token;
                    while($this->hasOperatorInStack()
                            && ($o2 = $this->operatorStack->top())
                            && $o1->hasLowerPriority($o2))
                    {
                        $this->outputQueue->enqueue($this->operatorStack->pop());
                    }

                    $this->operatorStack->push($o1);
                    break;
                case Token::T_LEFT_BRACKET:
                    $this->operatorStack->push($token);
                    break;
                case Token::T_RIGHT_BRACKET:
                    while((!$this->operatorStack->isEmpty()) && (Token::T_LEFT_BRACKET != $this->operatorStack->top()->getType())) {
                        $this->outputQueue->enqueue($this->operatorStack->pop());
                    }
                    if($this->operatorStack->isEmpty()) {
                        throw new InvalidArgumentException(sprintf('Mismatched parentheses: %s', implode(' ',$tokens)));
                    }
                    $this->operatorStack->pop();
                    break;
                default:
                    throw new InvalidArgumentException(sprintf('Invalid token detected: %s', $token));
                    break;
            }
        }
        while($this->hasOperatorInStack()) {
            $this->outputQueue->enqueue($this->operatorStack->pop());
        }

        if(!$this->operatorStack->isEmpty()) {
            throw new InvalidArgumentException(sprintf('Mismatched parenthesis or misplaced number: %s', implode(' ',$tokens)));
        }

        return iterator_to_array($this->outputQueue);
    }
394907
public function parse(array $tokens = array())
    {
        $this->lineNum    = -1;
        $this->lineTokens = 0;
        $this->pragmas    = $this->defaultPragmas;

        $this->pragmaFilters = isset($this->pragmas[Mustache_Engine::PRAGMA_FILTERS]);
        $this->pragmaBlocks  = isset($this->pragmas[Mustache_Engine::PRAGMA_BLOCKS]);

        return $this->buildTree($tokens);
    }
222832
public function parse(array $tokens) : int
    {
        $tokensAvailable = array_flip($this->getGrammar()->getTokens());

        $length = count($tokens);

        if ($length === 0) {
            throw new Exception('Invalid Roman', Exception::INVALID_ROMAN);
        }

        foreach ($tokens as $position => $token) {
            if (! is_string($token)) {
                $exception = new Exception(
                    sprintf('Invalid token type "%s" at position %d', gettype($token), $position),
                    Exception::INVALID_TOKEN_TYPE
                );

                $exception->setPosition($position);

                throw $exception;
            }

            if (! isset($tokensAvailable[$token])) {
                $exception = new Exception(
                    sprintf('Unknown token "%s" at position %d', $token, $position),
                    Exception::UNKNOWN_TOKEN
                );

                $exception
                    ->setToken($token)
                    ->setPosition($position);

                throw $exception;
            }
        }

        return (new Automaton($this->getGrammar()))->read($tokens)->getValue();
    }
1237904
protected function __add( array $tokens )
	{
		$updated = false;

		foreach ( $tokens as $token ) {
			if ( ! $this->__contains( $token ) ) {
				$this->tokens[] = $token;
				$updated = true;
			}
		}

		if ( $updated ) {
			$this->__update();
		}

		return $this;
	}
41935
static function parseUseTrait(array $tokens, &$i) {
    ++$i;
    $names = ParserUtil::parseIdentifierList($tokens, $i);
    if (FALSE === $names) {
      return FALSE;
    }
    $id = ParserUtil::nextSubstantialIncl($tokens, $i);
    if (';' === $id) {
      ++$i;
      return new AstUseTrait($names);
    }
    elseif ('{' === $id) {
      ParserUtil::skipCurly($tokens, $i);
      return new AstUseTrait($names);
    }
    else {
      return FALSE;
    }
  }
161809
public static function create(array $tokens) {
        HashMap::assert($tokens, 'tokens');

        if (empty($tokens['LIMIT'])) return [];

        $headers = [];
        array_push($headers, 'Limit: ' . $tokens['LIMIT']['rowcount']);
        if(empty($tokens['LIMIT']['offset'])) return $headers;
        array_push($headers, 'Offset: ' . $tokens['LIMIT']['offset']);
        return $headers;
    }
86410
private function parseTokens(array $tokens)
    {
        return array_map(function ($token, $key) {
            return ['name' => $key, 'value' => $token];
        }, $tokens, array_keys($tokens));
    }
997951
private function normalizeTokens(array $tokens)
    {
        $nTokens = array();
        for ($i=0,$c=count($tokens); $i<$c; $i++) {
            $token = $tokens[$i];

            if (is_string($token)) {
                $nTokens[] = new LiteralToken($token, end($nTokens)->getEndLine());
                continue;
            }

            switch ($token[0]) {
                case T_WHITESPACE:
                    $lines = explode("\n", $token[1]);
                    for ($j=0,$k=count($lines); $j<$k; $j++) {
                        $line = $lines[$j].($j+1 === $k ? '' : "\n");

                        if ($j+1 === $k && '' === $line) {
                            break;
                        }

                        $nTokens[] = new PhpToken(array(T_WHITESPACE, $line, $token[2] + $j));
                    }
                    break;

                default:
                    // remove any trailing whitespace of the token
                    if (preg_match('/^(.*?)(\s+)$/', $token[1], $match)) {
                        $nTokens[] = new PhpToken(array($token[0], $match[1], $token[2]));

                        // if the next token is whitespace, change it
                        if (isset($tokens[$i+1]) && !is_string($tokens[$i+1])
                                && T_WHITESPACE === $tokens[$i+1][0]) {
                            $tokens[$i+1][1] = $match[2].$tokens[$i+1][1];
                            $tokens[$i+1][2] = $token[2];
                        } else {
                            $nTokens[] = new PhpToken(array(T_WHITESPACE, $match[2], $token[2]));
                        }
                    } else {
                        $nTokens[] = new PhpToken($token);
                    }
            }
        }

        return $nTokens;
    }
608547
protected function filterTokens(array $tokens)
    {
        $filteredTokens = array_filter($tokens, function (Token $t) {
            return $t->getType() !== TokenType::Whitespace;
        });

        // Return the array values only, because array_filter preserves the keys
        return array_values($filteredTokens);
    }
1002602
protected function parseCreateIndex(SQLTokenIterator $tokens)
    {

        /* @var $entity CreateIndexStatement */
        $entity = new CreateIndexStatement();

        ### FLAGS

        if ($tokens->isToken(SqlToken::T_UNIQUE(), TokenIterator::PREVIOUS, [SqlToken::T_PRIMARY()])) {
            $entity->setIsUnique(true);
        }

        if ($tokens->isToken(SqlToken::T_PRIMARY(), TokenIterator::PREVIOUS, [SqlToken::T_UNIQUE()])) {
            $entity->setIsPrimary(true);
        }

        ### NAME

        if (!$tokens->seekTokenNum(T_STRING)) {
            throw new MalformedSqlException("Missing valid index-name!", $tokens);
        }

        $entity->setName($tokens->getCurrentTokenString());

        ### USING

        if ($tokens->seekTokenNum(SqlToken::T_USING())) {
            switch(true){

                // TODO: R-TREE index not implemented yet!
                case $tokens->seekTokenNum(SqlToken::T_RTREE()):
                case $tokens->seekTokenNum(SqlToken::T_BTREE()):
                    $entity->setIndexType(IndexType::BTREE());
                    break;

                case $tokens->seekTokenNum(SqlToken::T_HASH()):
                    $entity->setIndexType(IndexType::HASH());
                    break;

                default:
                    throw new MalformedSqlException("Invalid index-type specified!", $tokens);
            }
        }

        ### TABLE

        if (!$tokens->seekTokenNum(SqlToken::T_ON())) {
            throw new MalformedSqlException("Missing T_ON for CREATE INDEX statement!", $tokens);
        }

        if (!$this->tableParser->canParseTokens($tokens)) {
            throw new MalformedSqlException("Missing valid table-specifier for CREATE INDEX statement!", $tokens);
        }

        $entity->setTable($this->tableParser->convertSqlToJob($tokens));

        ### COLUMNS

        if (!$tokens->seekTokenText('(')) {
            throw new MalformedSqlException("Missing beginning parenthesis holding columns in CREATE INDEX statement!", $tokens);
        }

        do {
            if (!$this->columnParser->canParseTokens($tokens)) {
                throw new MalformedSqlException("Missing valid column-specifier in CREATE INDEX statement!", $tokens);
            }

            $column = $this->columnParser->convertSqlToJob($tokens);

            $length = null;
            if ($tokens->seekTokenText('(')) {
                if (!$this->valueParser->canParseTokens($tokens)) {
                    throw new MalformedSqlException("Missing valid column-length in CREATE INDEX statement!", $tokens);
                }

                $length = $this->valueParser->convertSqlToJob($tokens);

                if (!$tokens->seekTokenText(')')) {
                    throw new MalformedSqlException("Missing closing parenthesis holding column-length in CREATE INDEX statement!", $tokens);
                }
            }

            $direction = null;
            if ($tokens->seekTokenNum(SqlToken::T_ASC())) {
                $direction = SqlToken::T_ASC();
            } elseif ($tokens->seekTokenNum(SqlToken::T_DESC())) {
                $direction = SqlToken::T_DESC();
            }

            $entity->addColumn($column, $length, $direction);

        } while ($tokens->seekTokenText(','));

        if (!$tokens->seekTokenText(')')) {
            throw new MalformedSqlException("Missing closing parenthesis holding columns in CREATE INDEX statement!", $tokens);
        }

        ### WITH PARSER

        if ($tokens->seekTokenNum(SqlToken::T_WITH())) {
            if (!$tokens->seekTokenNum(SqlToken::T_PARSER())) {
                throw new MalformedSqlException("Missing T_PARSER after T_WITH in CREATE INDEX statement!", $tokens);
            }

            if (!$tokens->seekTokenNum(T_STRING)) {
                throw new MalformedSqlException("Missing valid parser name after WITH PARSER in CREATE INDEX statement!", $tokens);
            }

            $entity->setParser($tokens->getCurrentTokenString());
        }

        return $entity;
    }
1302955
public function remove(string ...$tokens)
    {
        if (count($tokens) === 0) {
            return;
        }
        if (count($this->tokens) === 0) {
            return;
        }
        foreach ($tokens as $t) {
            $i = array_search($t, $this->tokens);
            if ($i === false) {
                continue;
            }
            array_splice($this->tokens, $i, 1);
        }
        $this->setAttributeValue();
    }
729480
protected function updateTokens(array $tokens)
    {
        $this->roles = null;
        $this->contextHash = null;

        if ($this->request === null) {
            return;
        }

        /** @var $token TokenInterface */
        foreach ($tokens as $token) {
            $token->updateCredentials($this->request);
        }

        $sessionDataContainer = $this->objectManager->get(SessionDataContainer::class);
        $sessionDataContainer->setSecurityTokens(array_merge($this->inactiveTokens, $this->activeTokens));
    }
922775
private function executeStarts(array $indexes): void
    {
        BlockInterpreter::executeStarts(
            $indexes,
            $this->getBlocks(),
            $this->getContext(),
            $this->getErrorAggregator()
        );
    }
854897
public function initialize(array &$docs)
    {
        $doc_keys = range(0,count($docs)-1);
        $topic_keys = range(0,$this->ntopics-1);

        // initialize the arrays
        $this->words_in_doc = array_fill_keys(
            $doc_keys,
            0
        );
        $this->words_in_topic = array_fill_keys(
            $topic_keys,
            0
        );
        $this->count_docs_topics = array_fill_keys(
            $doc_keys,
            array_fill_keys(
                $topic_keys,
                0
            )
        );
        $this->count_topics_words = array_fill_keys(
            $topic_keys,
            array()
        );
        $this->word_doc_assigned_topic = array_fill_keys(
            $doc_keys,
            array()
        );
        $this->voc = array();

        foreach ($docs as $i=>$doc) {
            $this->words_in_doc[$i] = count($doc);
            foreach ($doc as $idx=>$w) {
                // choose a topic randomly to assign this word to
                $topic = (int) ($this->mt->generate()*$this->ntopics);

                //$this->words_in_doc[$i]++;
                $this->words_in_topic[$topic]++;
                $this->count_docs_topics[$i][$topic]++;

                if (!isset($this->count_topics_words[$topic][$w]))
                    $this->count_topics_words[$topic][$w]=0;
                $this->count_topics_words[$topic][$w]++;

                $this->word_doc_assigned_topic[$i][$idx] = $topic;

                $this->voc[$w] = 1;
            }
        }
        $this->voccnt = count($this->voc);
        $this->voc = array_keys($this->voc);
    }