Generates HTML from an array of tokens.
@param HTMLPurifier_Token[] $tokens Array of HTMLPurifier_Token
@return string Generated HTML
973439
public function generateFromTokens($tokens)
    {
        if (!$tokens) {
            return '';
        }

        // Basic algorithm
        $html = '';
        for ($i = 0, $size = count($tokens); $i < $size; $i++) {
            if ($this->_scriptFix && $tokens[$i]->name === 'script'
                && $i + 2 < $size && $tokens[$i+2] instanceof HTMLPurifier_Token_End) {
                // script special case
                // the contents of the script block must be ONE token
                // for this to work.
                $html .= $this->generateFromToken($tokens[$i++]);
                $html .= $this->generateScriptFromToken($tokens[$i++]);
            }
            $html .= $this->generateFromToken($tokens[$i]);
        }

        // Tidy cleanup
        if (extension_loaded('tidy') && $this->config->get('Output.TidyFormat')) {
            $tidy = new Tidy;
            $tidy->parseString(
                $html,
                array(
                   'indent'=> true,
                   'output-xhtml' => $this->_xhtml,
                   'show-body-only' => true,
                   'indent-spaces' => 2,
                   'wrap' => 68,
                ),
                'utf8'
            );
            $tidy->cleanRepair();
            $html = (string) $tidy; // explicit cast necessary
        }

        // Normalize newlines to system defined value
        if ($this->config->get('Core.NormalizeNewlines')) {
            $nl = $this->config->get('Output.Newline');
            if ($nl === null) {
                $nl = PHP_EOL;
            }
            if ($nl !== "\n") {
                $html = str_replace("\n", $nl, $html);
            }
        }
        return $html;
    }
448134
public function apply(array &$tokens)
		{
		$r = 0;
		for ($i = 0, $l = count($tokens); $i < $l; $i++)
			{
			if (get_class($tokens[$i]) === "CssCommentToken")
				{
				$tokens[$i] = null;
				$r++;
				}
			}
		return $r;
		}
1301945
public static function label($tokenId, array $tokens) {
		$token = $tokens[$tokenId];
		$hasName = in_array($token['id'], array(
			T_FUNCTION,
			T_CLASS,
			T_INTERFACE,
			T_VARIABLE,
		));
		if ($hasName) {
			if ($token['id'] === T_VARIABLE) {
				return substr($token['content'], 1);
			}
			$total = count($tokens);
			for ($key = $tokenId; $key <= $total; $key++) {
				if ($tokens[$key]['id'] === T_STRING) {
					return $tokens[$key]['content'];
				} elseif (in_array($tokens[$key]['content'], array('(', '{', ':'))) {
					break;
				}
			}
		}
		return null;
	}
850778
private function grabAttribsBeforeMethodToken(Tokens $tokens, $index)
    {
        static $tokenAttribsMap = [
            T_PRIVATE   => 'visibility',
            T_PROTECTED => 'visibility',
            T_PUBLIC    => null, // destroy T_PUBLIC token. This is literally the only change from the original
            T_ABSTRACT  => 'abstract',
            T_FINAL     => 'final',
            T_STATIC    => 'static',
        ];

        return $this->grabAttribsBeforeToken(
            $tokens,
            $index,
            $tokenAttribsMap,
            [
                'abstract'   => null,
                'final'      => null,
                'visibility' => null,
                'static'     => null,
            ]
        );
    }
24811
public static function createPagination(array $tokens, array $options) {
        if(!isset($options['pagination_as_query']) || !$options['pagination_as_query']) return '';

        $perPageParam = isset($options['per_page_param']) ? $options['per_page_param'] : PaginationQuery::DEFAULT_PER_PAGE_PARAM;
        $pageParam    = isset($options['page_param']) ? $options['page_param'] : PaginationQuery::DEFAULT_PAGE_PARAM;

        $paginationParameters = PaginationQuery::create($tokens, $perPageParam, $pageParam);

        return $paginationParameters ? http_build_query($paginationParameters) : '';
    }
250427
protected function formatOpeningParenthesis($token, $i, array &$tokens, array &$originalTokens)
    {
        $length = 0;
        for ($j = 1; $j <= 250; ++$j) {
            if (isset($tokens[$i + $j])) {
                $next = $tokens[$i + $j];
                if ($this->parentheses->stringIsClosingParentheses($next)) {
                    $this->parentheses->writeNewInlineParentheses();
                    break;
                }

                if ($this->parentheses->invalidParenthesesTokenValue($next)
                    || $this->parentheses->invalidParenthesesTokenType($next)
                ) {
                    break;
                }

                $length += \strlen($next[Tokenizer::TOKEN_VALUE]);
            }
        }
        $this->newLine->writeNewLineForLongInlineValues($length);

        if (WhiteSpace::isPrecedingCurrentTokenOfTokenTypeWhiteSpace($originalTokens, $token)) {
            $this->formattedSql = \rtrim($this->formattedSql, ' ');
        }

        $this->newLine->addNewLineAfterOpeningParentheses();

        return $tokens;
    }
1261210
public function render(array $tokens, $view, array $partials = null)
    {
        // Do some pre-initialization of variables used later in the routine
        $renderer = $this;
        $inLoop   = false;

        if (is_scalar($view)) {
            // Iteration over lists will sometimes involve scalars
            $inLoop = true;
        }

        if (null === $partials) {
            $partials = [];
        }

        $rendered = '';
        foreach ($tokens as $token) {
            list($type, $data) = $token;
            if ($value = $this->handlePragmas($token, $view)) {
                $rendered .= $value;
                continue;
            }
            switch ($type) {
                case Lexer::TOKEN_CONTENT:
                    $rendered .= $data;
                    break;
                case Lexer::TOKEN_VARIABLE:
                    $value = $this->getValue($data, $view);
                    if (is_scalar($value)) {
                        if ($test = $this->handlePragmas($token, $value)) {
                            $rendered .= $test;
                            break;
                        }

                        $rendered .= ('' === $value) ? '' : $this->escape($value);
                        break;
                    }

                    $pragmaView = [$data => $value];
                    if ($test = $this->handlePragmas($token, $pragmaView)) {
                        $rendered .= $test;
                        break;
                    }

                    $rendered .= (string) $value;
                    break;
                case Lexer::TOKEN_VARIABLE_RAW:
                    $value = $this->getValue($data, $view);
                    $rendered .= $value;
                    break;
                case Lexer::TOKEN_SECTION:
                    if ($inLoop) {
                        // In a loop, with scalar values; skip
                        break;
                    }
                    $section = $this->getValue($data['name'], $view);
                    if (! $section) {
                        // Section is not a true value; skip
                        break;
                    }

                    // Build the section view
                    $sectionView = $section;
                    if (is_bool($section)) {
                        // For a boolean true, pass the current view
                        $sectionView = $view;
                    }
                    if (is_array($section) || $section instanceof Traversable) {
                        if (is_array($section) && $this->isAssocArray($section)) {
                            // Nested view; pass it as the view
                            $sectionView = $section;
                        } else {
                            // Iteration
                            $renderedSection = '';
                            foreach ($section as $sectionView) {
                                $renderedSection .= $this->render($data['content'], $sectionView);
                            }
                            $rendered .= $renderedSection;
                            break;
                        }
                    } elseif (is_callable($section) && $this->isValidCallback($section)) {
                        // Higher order section
                        // Execute the callback, passing it the section's template
                        // string, as well as a renderer lambda.
                        $mustache       = $this->mustache;
                        $invokedPragmas = $this->invokedPragmas;
                        $rendered .= call_user_func(
                            $section,
                            $data['template'],
                            function ($text) use ($renderer, $mustache, $view, $partials) {
                                $tokens = $mustache->tokenize($text);
                                return $renderer->render($tokens, $view, $partials);
                            }
                        );
                        $this->registerPragmas($invokedPragmas);
                        break;
                    } elseif (is_object($section)) {
                        // In this case, the child object is the view.
                        $sectionView = $section;
                    } else {
                        // All other types, simply pass the current view
                        $sectionView = $view;
                    }

                    // Render the section
                    $invokedPragmas = $this->invokedPragmas;
                    $rendered .= $this->render($data['content'], $sectionView);
                    $this->registerPragmas($invokedPragmas);
                    break;
                case Lexer::TOKEN_SECTION_INVERT:
                    if ($inLoop) {
                        // In a loop, with scalar values; skip
                        break;
                    }
                    $section = $this->getValue($data['name'], $view);
                    if ($section) {
                        // If a value exists for the section, we skip it
                        $rendered .= '';
                        break;
                    }

                    // Otherwise, we render it
                    $invokedPragmas = $this->invokedPragmas;
                    $rendered .= $this->render($data['content'], $view);
                    $this->registerPragmas($invokedPragmas);
                    break;
                case Lexer::TOKEN_PLACEHOLDER:
                    if ($inLoop) {
                        // In a loop, with scalar values; skip
                        break;
                    }
                    $rendered .= $this->render($data['content'], $view);
                    break;
                case Lexer::TOKEN_PARTIAL:
                    if ($inLoop) {
                        // In a loop, with scalar values; skip
                        break;
                    }
                    $invokedPragmas = $this->invokedPragmas;
                    $this->clearInvokedPragmas();
                    if (! isset($data['tokens'])) {
                        $name = $data['partial'];
                        if (isset($partials[$data['partial']])) {
                            // Partial invoked is an aliased partial
                            $rendered .= $this->render($partials[$data['partial']], $view);
                        } else {
                            $partialTokens = $this->mustache->tokenize($data['partial']);
                            $rendered .= $this->render($partialTokens, $view);
                        }
                        $this->registerPragmas($invokedPragmas);
                        break;
                    }
                    $rendered .= $this->render($data['tokens'], $view);
                    $this->registerPragmas($invokedPragmas);
                    break;
                case Lexer::TOKEN_PRAGMA:
                    $this->registerPragma($data);
                    break;
                case Lexer::TOKEN_DELIM_SET:
                case Lexer::TOKEN_COMMENT:
                default:
                    // do nothing; only necessary for tokenization/parsing
                    break;
            }
        }
        return $rendered;
    }
568194
protected function fragment(array $tokens): string
    {
        $data = Arr::map($tokens['tokens'] ?? $tokens, function (Token $token) {
            return $token->token;
        });

        return \str_replace(
            '. ',
            '.',
            \implode(' ', $data)
        );
    }
1131442
public function apply(array &$tokens)
		{
		for ($i = 0, $l = count($tokens); $i < $l; $i++)
			{
			$current	= get_class($tokens[$i]);
			$next		= isset($tokens[$i+1]) ? get_class($tokens[$i+1]) : false;
			if (($current === "CssRulesetDeclarationToken" && $next === "CssRulesetEndToken") ||
				($current === "CssAtFontFaceDeclarationToken" && $next === "CssAtFontFaceEndToken") || 
				($current === "CssAtPageDeclarationToken" && $next === "CssAtPageEndToken"))
				{
				$tokens[$i]->IsLast = true;
				}
			}
		return 0;
		}
626676
public static function createConditionals(array $tokens) {
        if(!isset($tokens['WHERE'])) return '';

        $tableAlias = Table::alias($tokens);
        $primaryKeyColumn = sprintf('%s.%s', $tableAlias, Identifier::column($tokens, new MetaData));

        // Get WHERE conditions as string including table alias and primary key column if present
        $sqlWhereString = array_reduce($tokens['WHERE'], function($query, $token) use ($tableAlias) {
            return $query . str_replace(['"', '\''], '', str_replace('OR', '|', str_replace('AND', '&', $token['base_expr'])));
        });

        // Remove primary key column before removing table alias and returning
        return str_replace($tableAlias . '.', '', preg_replace('/' . preg_quote($primaryKeyColumn) . '=[\w\d]*&*/', '', $sqlWhereString));
    }
898852
private function extractUseStatement(\ArrayIterator $tokens)
    {
        $result = [''];
        while ($tokens->valid()
            && (self::T_LITERAL_USE_SEPARATOR !== $tokens->current()[0])
            && (self::T_LITERAL_END_OF_USE !== $tokens->current()[0])
        ) {
            if (T_AS === $tokens->current()[0]) {
                $result[] = '';
            }
            if (T_STRING === $tokens->current()[0] || T_NS_SEPARATOR === $tokens->current()[0]) {
                $result[\count($result) - 1] .= $tokens->current()[1];
            }
            $tokens->next();
        }

        if (1 == \count($result)) {
            $backslashPos = strrpos($result[0], '\\');

            if (false !== $backslashPos) {
                $result[] = substr($result[0], $backslashPos + 1);
            } else {
                $result[] = $result[0];
            }
        }

        return array_reverse($result);
    }
801227
protected static function _tokenize(array $tokens) {
		$inString = false;
		$results = array();
		$cpt = 0;
		$previousTokenId = null;
		foreach ($tokens as $tokenId => $token) {
			if ($token['content'] === '"') {
				if (!$inString) {
					$token['id'] = T_START_DOUBLE_QUOTE;
					$token['name'] = 'T_START_DOUBLE_QUOTE';
				} else {
					$token['id'] = T_END_DOUBLE_QUOTE;
					$token['name'] = 'T_END_DOUBLE_QUOTE';
				}
				$inString = !$inString;
			}

			$isCurlyBrace = (
				$token['content'] === '$' &&
				isset($tokens[$tokenId + 1]) &&
				$tokens[$tokenId + 1]['content'] === '{'
			);
			if ($isCurlyBrace) {
				$token['id'] = T_DOLLAR_CURLY_BRACES;
				$token['name'] = 'T_DOLLAR_CURLY_BRACES';
			}

			$isArray = (
				!$token['id'] &&
				$token['content'] === '(' &&
				$previousTokenId !== null &&
				$tokens[$previousTokenId]['id'] === T_ARRAY
			);
			if ($isArray) {
				$token['id'] = T_ARRAY_OPEN;
				$token['name'] = 'T_ARRAY_OPEN';
			}

			$isShortArray = (
				!$token['id'] &&
				$token['content'] === '[' &&
				$previousTokenId !== null && (
					$tokens[$previousTokenId]['id'] !== T_VARIABLE &&
					$tokens[$previousTokenId]['id'] !== T_ENCAPSED_AND_WHITESPACE
				)
			);
			if ($isShortArray) {
				$token['id'] = T_SHORT_ARRAY_OPEN;
				$token['name'] = 'T_SHORT_ARRAY_OPEN';
			}

			if ($token['content'] === '(' && !$token['id']) {
				$token['id'] = T_START_BRACKET;
				$token['name'] = 'T_START_BRACKET';
			}

			if ($token['id'] !== T_WHITESPACE) {
				$previousTokenId = $tokenId;
			}

			if ($token['id'] === T_IF) {
				$i = $cpt;
				$spaces = '';
				while ($i-- > 0 && $results[$i]['id'] === T_WHITESPACE) {
					$spaces = $results[$i]['content'] . $spaces;
				}
				if ($i >= 0 && $results[$i]['id'] === T_ELSE) {
					$token['id'] = T_ELSEIF;
					$token['name'] = 'T_ELSEIF';
					$token['content'] = $results[$i]['content'] . $spaces . $token['content'];
					$cpt = $i;
				}
			}

			$results[$cpt++] = $token;
		}

		return $results;
	}
608547
protected function filterTokens(array $tokens)
    {
        $filteredTokens = array_filter($tokens, function (Token $t) {
            return $t->getType() !== TokenType::Whitespace;
        });

        // Return the array values only, because array_filter preserves the keys
        return array_values($filteredTokens);
    }
594549
protected function getNamedCapturesExpressions(array $tokens)
	{
		$exprs = [];
		foreach ($tokens as $token)
		{
			if ($token['type'] !== 'capturingSubpatternStart' || !isset($token['name']))
			{
				continue;
			}

			$expr = $token['content'];
			if (strpos($expr, '|') !== false)
			{
				$expr = '(?:' . $expr . ')';
			}

			$exprs[$token['name']] = $expr;
		}

		return $exprs;
	}
1144896
protected function compileWhere(array $tokens): string
    {
        if (empty($tokens)) {
            return '';
        }

        $statement = '';

        $activeGroup = true;
        foreach ($tokens as $condition) {
            //OR/AND keyword
            $boolean = $condition[0];

            //See AbstractWhere
            $context = $condition[1];

            //First condition in group/query, no any AND, OR required
            if ($activeGroup) {
                //Kill AND, OR and etc.
                $boolean = '';

                //Next conditions require AND or OR
                $activeGroup = false;
            }

            /*
             * When context is string it usually represent control keyword/syntax such as opening
             * or closing braces.
             */
            if (is_string($context)) {
                if ($context == '(') {
                    //New where group.
                    $activeGroup = true;
                }

                $statement .= ltrim("{$boolean} {$context} ");

                if ($context == ')') {
                    //We don't need trailing space
                    $statement = rtrim($statement);
                }

                continue;
            }

            if ($context instanceof FragmentInterface) {
                //Fragments has to be compiled separately
                $statement .= "{$boolean} {$this->prepareFragment($context)} ";
                continue;
            }

            //Now we are operating with "class" where function, where we need 3 variables where(id, =, 1)
            if (!is_array($context)) {
                throw new CompilerException('Invalid where token, context expected to be an array');
            }

            /*
             * This is "normal" where token which includes identifier, operator and value.
             */
            list($identifier, $operator, $value) = $context;

            //Identifier can be column name, expression or even query builder
            $identifier = $this->quote($identifier);

            //Value has to be prepared as well
            $placeholder = $this->prepareValue($value);

            if ($operator == 'BETWEEN' || $operator == 'NOT BETWEEN') {
                //Between statement has additional parameter
                $right = $this->prepareValue($context[3]);

                $statement .= "{$boolean} {$identifier} {$operator} {$placeholder} AND {$right} ";
                continue;
            }

            //Compiler can switch equal to IN if value points to array (do we need it?)
            $operator = $this->prepareOperator($value, $operator);

            $statement .= "{$boolean} {$identifier} {$operator} {$placeholder} ";
        }

        if ($activeGroup) {
            throw new CompilerException('Unable to build where statement, unclosed where group');
        }

        return trim($statement);
    }
930957
public function renderInline(Token $token) {
        $alt = $token->get('alt', '');
        $url = $token->get('url', '');
        $title = $token->get('title', '');

        if (empty($title)) {
            return "<img src=\"$url\" alt=\"$alt\" />";
        }

        return "<img src=\"$url\" alt=\"$alt\" title=\"$title\" />";
    }
41935
static function parseUseTrait(array $tokens, &$i) {
    ++$i;
    $names = ParserUtil::parseIdentifierList($tokens, $i);
    if (FALSE === $names) {
      return FALSE;
    }
    $id = ParserUtil::nextSubstantialIncl($tokens, $i);
    if (';' === $id) {
      ++$i;
      return new AstUseTrait($names);
    }
    elseif ('{' === $id) {
      ParserUtil::skipCurly($tokens, $i);
      return new AstUseTrait($names);
    }
    else {
      return FALSE;
    }
  }