Render token list

@param \ViKon\Parser\TokenList $tokenList parsed token list
@param string                  $skin      renderer skin

@return bool|string FALSE on failure otherwise output

@throws \ViKon\Parser\ParserException
979724
public function render(TokenList $tokenList, $skin = 'default') {
        if (!array_key_exists($skin, $this->ruleRenderers)) {
            throw new ParserException('Skin ' . $skin . ' not found');
        }

        if (!array_key_exists($skin, $this->tokenRenderers)) {
            $this->tokenRenderers[$skin] = [];
            foreach ($this->ruleRenderers[$skin] as $ruleRenderer) {
                $ruleRenderer->register($this);
            }
        }

        \Event::fire('vikon.parser.before.render', [$tokenList]);

        $output = '';

        foreach ($tokenList->getTokens() as $token) {
            \Event::fire('vikon.parser.token.render.' . $token->getName(), [$token, $tokenList]);

            if (array_key_exists($token->getName(), $this->tokenRenderers[$skin])) {
                $output .= $this->tokenRenderers[$skin][$token->getName()]($token, $tokenList);
            } else {
                $output .= (string)$token;
            }
        }

        return $output;
    }
1284075
public static function printTokens(
        InputInterface $input,
        OutputInterface $output,
        array $tokens
    ) {
        $withMetadata = $input->getOption('with-metadata');
        $table = new Table($output);
        $headers = ['UUID', 'Indices', 'endpoints', 'plugins', 'ttl'];
        if ($withMetadata) {
            $headers[] = 'Metadata';
        }
        $table->setHeaders($headers);

        foreach ($tokens as $token) {
            $row = [
                $token->getTokenUUID()->composeUUID(),
                implode(', ', array_map(function (IndexUUID $index) {
                    return $index->composeUUID();
                }, $token->getIndices())),
                implode(', ', $token->getEndpoints()),
                implode(', ', $token->getPlugins()),
                $token->getTtl(),
            ];

            if ($withMetadata) {
                $metadataRow = [];
                foreach ($token->getMetadata() as $metadataField => $metadataValue) {
                    $metadataRow[] = sprintf("$metadataField = %s",
                        is_bool($metadataValue)
                            ? $metadataValue ? 'true' : 'false'
                            : (
                                is_array($metadataValue)
                                    ? implode(', ', $metadataValue)
                                    : (string) $metadataValue
                            )
                    );
                }
                $row[] = implode(PHP_EOL, $metadataRow);
            }

            $table->addRow($row);
        }

        $table->render();
    }
716620
static function parseIdentifierList(array $tokens, &$i) {
    $list = array();
    while (TRUE) {
      $id = ParserUtil::nextSubstantialIncl($tokens, $i);
      if (T_STRING !== $id) {
        return FALSE;
      }
      $list[] = $tokens[$i][1];
      $id = ParserUtil::nextSubstantialExcl($tokens, $i);
      if (',' !== $id) {
        break;
      }
      ++$i;
    }
    return $list;
  }
597559
public static function parse(Parser $parser, TokensList $list, array $options = [])
    {
        $ret = [];

        $expr = new self();

        /**
         * Counts brackets.
         *
         * @var int
         */
        $brackets = 0;

        /**
         * Whether there was a `BETWEEN` keyword before or not.
         *
         * It is required to keep track of them because their structure contains
         * the keyword `AND`, which is also an operator that delimits
         * expressions.
         *
         * @var bool
         */
        $betweenBefore = false;

        for (; $list->idx < $list->count; ++$list->idx) {
            /**
             * Token parsed at this moment.
             *
             * @var Token
             */
            $token = $list->tokens[$list->idx];

            // End of statement.
            if ($token->type === Token::TYPE_DELIMITER) {
                break;
            }

            // Skipping whitespaces and comments.
            if ($token->type === Token::TYPE_COMMENT) {
                continue;
            }

            // Replacing all whitespaces (new lines, tabs, etc.) with a single
            // space character.
            if ($token->type === Token::TYPE_WHITESPACE) {
                $expr->expr .= ' ';
                continue;
            }

            // Conditions are delimited by logical operators.
            if (in_array($token->value, static::$DELIMITERS, true)) {
                if ($betweenBefore && ($token->value === 'AND')) {
                    // The syntax of keyword `BETWEEN` is hard-coded.
                    $betweenBefore = false;
                } else {
                    // The expression ended.
                    $expr->expr = trim($expr->expr);
                    if (! empty($expr->expr)) {
                        $ret[] = $expr;
                    }

                    // Adding the operator.
                    $expr = new self($token->value);
                    $expr->isOperator = true;
                    $ret[] = $expr;

                    // Preparing to parse another condition.
                    $expr = new self();
                    continue;
                }
            }

            if (($token->type === Token::TYPE_KEYWORD)
                && ($token->flags & Token::FLAG_KEYWORD_RESERVED)
                && ! ($token->flags & Token::FLAG_KEYWORD_FUNCTION)
            ) {
                if ($token->value === 'BETWEEN') {
                    $betweenBefore = true;
                }
                if (($brackets === 0) && empty(static::$ALLOWED_KEYWORDS[$token->value])) {
                    break;
                }
            }

            if ($token->type === Token::TYPE_OPERATOR) {
                if ($token->value === '(') {
                    ++$brackets;
                } elseif ($token->value === ')') {
                    if ($brackets === 0) {
                        break;
                    }
                    --$brackets;
                }
            }

            $expr->expr .= $token->token;
            if (($token->type === Token::TYPE_NONE)
                || (($token->type === Token::TYPE_KEYWORD)
                && (! ($token->flags & Token::FLAG_KEYWORD_RESERVED)))
                || ($token->type === Token::TYPE_STRING)
                || ($token->type === Token::TYPE_SYMBOL)
            ) {
                if (! in_array($token->value, $expr->identifiers)) {
                    $expr->identifiers[] = $token->value;
                }
            }
        }

        // Last iteration was not processed.
        $expr->expr = trim($expr->expr);
        if (! empty($expr->expr)) {
            $ret[] = $expr;
        }

        --$list->idx;

        return $ret;
    }
1200352
public function parse($tokens = null, $additional = [], $embedded = false)
    {
        if (is_string($tokens)) {
            $tokens = $this->tokenize($tokens, $additional, $embedded);
        } elseif (!$tokens instanceof TokenIterator) {
            // Todo: Own Exceptions
            throw new \InvalidArgumentException('$tokens must be string or TokenIterator');
        }

        return $this->_process($tokens);
    }
678651
public static function parse(Parser $parser, TokensList $list, array $options = [])
    {
        $ret = new self();

        /**
         * The ID that will be assigned to duplicate options.
         *
         * @var int
         */
        $lastAssignedId = count($options) + 1;

        /**
         * The option that was processed last time.
         *
         * @var array
         */
        $lastOption = null;

        /**
         * The index of the option that was processed last time.
         *
         * @var int
         */
        $lastOptionId = 0;

        /**
         * Counts brackets.
         *
         * @var int
         */
        $brackets = 0;

        /**
         * The state of the parser.
         *
         * Below are the states of the parser.
         *
         *      0 ---------------------[ option ]----------------------> 1
         *
         *      1 -------------------[ = (optional) ]------------------> 2
         *
         *      2 ----------------------[ value ]----------------------> 0
         *
         * @var int
         */
        $state = 0;

        for (; $list->idx < $list->count; ++$list->idx) {
            /**
             * Token parsed at this moment.
             *
             * @var Token
             */
            $token = $list->tokens[$list->idx];

            // End of statement.
            if ($token->type === Token::TYPE_DELIMITER) {
                break;
            }

            // Skipping comments.
            if ($token->type === Token::TYPE_COMMENT) {
                continue;
            }

            // Skipping whitespace if not parsing value.
            if (($token->type === Token::TYPE_WHITESPACE) && ($brackets === 0)) {
                continue;
            }

            if ($lastOption === null) {
                $upper = strtoupper($token->token);
                if (isset($options[$upper])) {
                    $lastOption = $options[$upper];
                    $lastOptionId = is_array($lastOption) ?
                        $lastOption[0] : $lastOption;
                    $state = 0;

                    // Checking for option conflicts.
                    // For example, in `SELECT` statements the keywords `ALL`
                    // and `DISTINCT` conflict and if used together, they
                    // produce an invalid query.
                    //
                    // Usually, tokens can be identified in the array by the
                    // option ID, but if conflicts occur, a generated option ID
                    // is used.
                    //
                    // The first pseudo duplicate ID is the maximum value of the
                    // real options (e.g.  if there are 5 options, the first
                    // fake ID is 6).
                    if (isset($ret->options[$lastOptionId])) {
                        $parser->error(
                            sprintf(
                                Translator::gettext('This option conflicts with "%1$s".'),
                                is_array($ret->options[$lastOptionId])
                                ? $ret->options[$lastOptionId]['name']
                                : $ret->options[$lastOptionId]
                            ),
                            $token
                        );
                        $lastOptionId = $lastAssignedId++;
                    }
                } else {
                    // There is no option to be processed.
                    break;
                }
            }

            if ($state === 0) {
                if (! is_array($lastOption)) {
                    // This is a just keyword option without any value.
                    // This is the beginning and the end of it.
                    $ret->options[$lastOptionId] = $token->value;
                    $lastOption = null;
                    $state = 0;
                } elseif (($lastOption[1] === 'var') || ($lastOption[1] === 'var=')) {
                    // This is a keyword that is followed by a value.
                    // This is only the beginning. The value is parsed in state
                    // 1 and 2. State 1 is used to skip the first equals sign
                    // and state 2 to parse the actual value.
                    $ret->options[$lastOptionId] = [
                        // @var string The name of the option.
                        'name' => $token->value,
                        // @var bool Whether it contains an equal sign.
                        //           This is used by the builder to rebuild it.
                        'equals' => $lastOption[1] === 'var=',
                        // @var string Raw value.
                        'expr' => '',
                        // @var string Processed value.
                        'value' => '',
                    ];
                    $state = 1;
                } elseif ($lastOption[1] === 'expr' || $lastOption[1] === 'expr=') {
                    // This is a keyword that is followed by an expression.
                    // The expression is used by the specialized parser.

                    // Skipping this option in order to parse the expression.
                    ++$list->idx;
                    $ret->options[$lastOptionId] = [
                        // @var string The name of the option.
                        'name' => $token->value,
                        // @var bool Whether it contains an equal sign.
                        //           This is used by the builder to rebuild it.
                        'equals' => $lastOption[1] === 'expr=',
                        // @var Expression The parsed expression.
                        'expr' => '',
                    ];
                    $state = 1;
                }
            } elseif ($state === 1) {
                $state = 2;
                if ($token->token === '=') {
                    $ret->options[$lastOptionId]['equals'] = true;
                    continue;
                }
            }

            // This is outside the `elseif` group above because the change might
            // change this iteration.
            if ($state === 2) {
                if ($lastOption[1] === 'expr' || $lastOption[1] === 'expr=') {
                    $ret->options[$lastOptionId]['expr'] = Expression::parse(
                        $parser,
                        $list,
                        empty($lastOption[2]) ? [] : $lastOption[2]
                    );
                    $ret->options[$lastOptionId]['value']
                        = $ret->options[$lastOptionId]['expr']->expr;
                    $lastOption = null;
                    $state = 0;
                } else {
                    if ($token->token === '(') {
                        ++$brackets;
                    } elseif ($token->token === ')') {
                        --$brackets;
                    }

                    $ret->options[$lastOptionId]['expr'] .= $token->token;

                    if (! ((($token->token === '(') && ($brackets === 1))
                        || (($token->token === ')') && ($brackets === 0)))
                    ) {
                        // First pair of brackets is being skipped.
                        $ret->options[$lastOptionId]['value'] .= $token->value;
                    }

                    // Checking if we finished parsing.
                    if ($brackets === 0) {
                        $lastOption = null;
                    }
                }
            }
        }

        /*
         * We reached the end of statement without getting a value
         * for an option for which a value was required
         */
        if ($state === 1
            && $lastOption
            && ($lastOption[1] === 'expr'
            || $lastOption[1] === 'var'
            || $lastOption[1] === 'var='
            || $lastOption[1] === 'expr=')
        ) {
            $parser->error(
                sprintf(
                    'Value/Expression for the option %1$s was expected.',
                    $ret->options[$lastOptionId]['name']
                ),
                $list->tokens[$list->idx - 1]
            );
        }

        if (empty($options['_UNSORTED'])) {
            ksort($ret->options);
        }

        --$list->idx;

        return $ret;
    }
295401
public function parseToken($content, $position, $state, TokenList $tokenList) {
        switch ($state) {
            case Lexer::STATE_MATCHED:
                $token = $tokenList->addToken($this->name, $position);
                $token->set('content', str_repeat("\n", substr_count($content, "\n")));
                break;
            default:
                parent::parseToken($content, $position, $state, $tokenList);
                break;
        }
    }
1282897
private function hasToken(array $matchList, string $tokenKey): bool
    {
        return array_key_exists($tokenKey, $matchList) && strlen($matchList[$tokenKey]);
    }
1332990
private function fileEnd(array $tokenList, int $nextLine, string $delimiterToken): bool
    {
        return $nextLine == count($tokenList) || $delimiterToken === $tokenList[$nextLine]->getType();
    }
336140
public function merge(TokenList $tokenList) {
        $tokenList->closeOpenTokens();
        $lastTokenPosition = $this->last() === null ? 0 : $this->last()
            ->getPosition();
        foreach ($tokenList->getTokens() as $token) {
            $token->setPosition($token->getPosition() + $lastTokenPosition);
            $this->tokens[] = $token;
        }
    }
350807
function renderToken(token) {
  var syntax = token.getCGSyntax();
  if (syntax in renderFunc === false) {
    throw new Error('Undefined syntax: ' + syntax);
  }

  return renderFunc[syntax](token.getBody(), this.MDRenderer);
}
144162
protected function handleSingleState($content, $position, TokenList $tokenList) {
        $content = trim($content);

        preg_match('/^#{1,6}/', $content, $matches);

        $tokenList->addToken($this->name, $position)
            ->set('level', abs(strlen($matches[0])))
            ->set('content', trim($content, "# \t\n\r\0\x0B"));

        return true;

    }
642379
private CommonTree render(TokenStream tokens, CommonTree tree, QueryRendererDelegate rendererDelegate) throws RecognitionException {
      CommonTreeNodeStream treeNodeStream = new CommonTreeNodeStream(tree);
      treeNodeStream.setTokenStream(tokens);
      return (CommonTree) new QueryRenderer(treeNodeStream, rendererDelegate).statement().getTree();
   }
75776
private function renderCsrfToken() {
		
		$tokenManager = new CsrfTokenManager();
		$token = $tokenManager->refreshToken('_' . $this->action . '_');

		$e = new InputElement(self::CSRF_TOKEN_NAME, $token->getValue());
		$e->setAttribute('type', 'hidden');
		
		return $e->render();
	}
916305
public static function parse(Parser $parser, TokensList $list, array $options = [])
    {
        $ret = new self();

        /**
         * The state of the parser.
         *
         * Below are the states of the parser.
         *
         *      0 ---------------- [ tbl_name ] -----------------> 1
         *      1 ---------------- [ lock_type ] ----------------> 2
         *      2 -------------------- [ , ] --------------------> break
         *
         * @var int
         */
        $state = 0;

        $prevToken = null;

        for (; $list->idx < $list->count; ++$list->idx) {
            /**
             * Token parsed at this moment.
             *
             * @var Token
             */
            $token = $list->tokens[$list->idx];

            // End of statement.
            if ($token->type === Token::TYPE_DELIMITER
                || ($token->type === Token::TYPE_OPERATOR
                && $token->value === ',')
            ) {
                break;
            }

            if ($state === 0) {
                $ret->table = Expression::parse($parser, $list, ['parseField' => 'table']);
                $state = 1;
            } elseif ($state === 1) {
                // parse lock type
                $ret->type = self::parseLockType($parser, $list);
                $state = 2;
            }
            $prevToken = $token;
        }

        // 2 is the only valid end state
        if ($state !== 2) {
            $parser->error('Unexpected end of LOCK expression.', $prevToken);
        }

        --$list->idx;

        return $ret;
    }
1227666
public static function parse(Parser $parser, TokensList $list, array $options = [])
    {
        $ret = new self();

        $offset = false;

        for (; $list->idx < $list->count; ++$list->idx) {
            /**
             * Token parsed at this moment.
             *
             * @var Token
             */
            $token = $list->tokens[$list->idx];

            // End of statement.
            if ($token->type === Token::TYPE_DELIMITER) {
                break;
            }

            // Skipping whitespaces and comments.
            if (($token->type === Token::TYPE_WHITESPACE) || ($token->type === Token::TYPE_COMMENT)) {
                continue;
            }

            if (($token->type === Token::TYPE_KEYWORD) && ($token->flags & Token::FLAG_KEYWORD_RESERVED)) {
                break;
            }

            if ($token->type === Token::TYPE_KEYWORD && $token->keyword === 'OFFSET') {
                if ($offset) {
                    $parser->error('An offset was expected.', $token);
                }
                $offset = true;
                continue;
            }

            if (($token->type === Token::TYPE_OPERATOR) && ($token->value === ',')) {
                $ret->offset = $ret->rowCount;
                $ret->rowCount = 0;
                continue;
            }

            if ($offset) {
                $ret->offset = $token->value;
                $offset = false;
            } else {
                $ret->rowCount = $token->value;
            }
        }

        if ($offset) {
            $parser->error(
                'An offset was expected.',
                $list->tokens[$list->idx - 1]
            );
        }

        --$list->idx;

        return $ret;
    }
269543
public function setToken($tokenId, CsrfToken $token) {
		
		if(!trim((string) $tokenId)) {
			throw new \InvalidArgumentException('Invalid token id.');
		}
		
		$this->sessionDataBag->set(
			$this->namespace . '/' . $tokenId,
			$token
		);
	}