Handles single token and passes it to a callback function if specified.

@param int|null $tokenType Token type.
@param string   $content   Non parsed token content.
486029
protected function handleToken($tokenType, string $content)
    {
        if ($tokenType == self::PLAIN_TEXT) {
            if (empty($content)) {
                return;
            }

            $token = [
                self::TOKEN_TYPE    => self::PLAIN_TEXT,
                self::TOKEN_CONTENT => $this->repairPHP($content)
            ];
        } else {
            $token = $this->parseToken($content);
        }

        $this->tokens[] = $token;
    }
308248
private function addParam($value, array $token)
    {
        if (!$this->currentLabel) {
            throw new TokenException(sprintf('parameter name missing in %s()', $this->name), $token);
        }

        $this->params[$this->currentLabel] = $value;

        $this->currentLabel = null;
    }
1077998
public function setSelectorTokenString(string $string)
    {
        if (strpos($string, ':') === false) {
            Throw new TokenException('AuthToken: The key is no valid selector:token string.');
        }
        $this->string = urldecode($string);

        $parts = explode(':', $this->string);

        $this->selector = $parts[0];
        $this->activation_token = $parts[1];
    }
878217
public function setContentType($contentType = null)
    {
        if ($contentType === null) {
            $contentType = isset($_SERVER['CONTENT_TYPE']) ? $_SERVER['CONTENT_TYPE'] : null;
        }

        $this->contentType = $contentType;
    }
499869
public function skipUntil($type)
    {
        while ( ! $this->isNext($type) && $this->moveNext());

        if ( ! $this->isNext($type)) {
            throw new \RuntimeException(sprintf('Could not find the token %s.', $this->getName($type)));
        }
    }
532
function indateto($content){
        if (strlen($content) < 8) { // State exit or missing parameter.
            return true;
        }
        // Strip off the dateto: part and add the reminder to the parsed token array
        $param = trim(substr($content,7));
        $this->tokens[] = new search_token(TOKEN_DATETO,$param);
        return true;
    }
144162
protected function handleSingleState($content, $position, TokenList $tokenList) {
        $content = trim($content);

        preg_match('/^#{1,6}/', $content, $matches);

        $tokenList->addToken($this->name, $position)
            ->set('level', abs(strlen($matches[0])))
            ->set('content', trim($content, "# \t\n\r\0\x0B"));

        return true;

    }
554367
public function setTokenType(TokenTypeInterface $tokenType)
    {
        $this->issuer->setTokenType($tokenType);
        $this->checker->setTokenType($tokenType);
    }
865210
public function enterToken(array $token, $content)
    {
        switch ($token[Lexer::TYPE]) {
            case Lexer::T_FUNC_CLOSE:
                return $this->parent;

            case Lexer::T_LABEL:
                $this->currentLabel = trim(substr(trim($token[Lexer::VALUE]), 0, -1));
                break;

            case Lexer::T_QUOTED_STRING:
            case Lexer::T_NUMBER:
            case Lexer::T_TRUE:
            case Lexer::T_FALSE:
            case Lexer::T_NULL:
                $value = $this->getValueOfToken($token);
                $this->addParam($value, $token);
                break;

            case Lexer::T_ARRAY_OPEN:
                $array = new ArrayContext($token, $this);
                $this->addParam($array, $token);

                return $array;
                break;

            case Lexer::T_FUNC_OPEN:
                $func = new FuncContext($token, $this);
                $this->addParam($func, $token);

                return $func;

            case Lexer::T_COMMA:
                break;

            default:
                return null;
        }

        return $this;
    }
1156289
function tokener(value, type) {
		session.tokens.push({
			value: value,
			type:  type || value,
			start: null,
			end:   null
		});
	}
530390
protected void parse( String content ) {
        Tokenizer tokenizer = new CndTokenizer(false, true);
        TokenStream tokens = new TokenStream(content, tokenizer, false);
        tokens.start();
        while (tokens.hasNext()) {
            // Keep reading while we can recognize one of the two types of statements ...
            if (tokens.matches("<", ANY_VALUE, "=", ANY_VALUE, ">")) {
                parseNamespaceMapping(tokens);
            } else if (tokens.matches("[", ANY_VALUE, "]")) {
                parseNodeTypeDefinition(tokens);
            } else {
                Position position = tokens.previousPosition();
                throw new ParsingException(position, CndI18n.expectedNamespaceOrNodeDefinition.text(tokens.consume(),
                                                                                                    position.getLine(),
                                                                                                    position.getColumn()));
            }
        }
    }
200042
public function contentType(string $content_type, string $charset = '')
    {
        $header = new ContentType($content_type);

        if (!empty($charset)) {
            $header->setCharset($charset);
        }

        $this->addHeader($header);
    }
1143555
protected function write($string, $token)
    {
        if ($this->stream === null) {
            echo $string;
        } else {
            $callback = $this->stream;
            $callback($string, $token);
        }

        $this->bytes += strlen($string);
    }
776750
protected function _setTokenEnd($delimiter)
    {
        if (!is_null($delimiter)) {
            $delimiter = $this->_normalizeTokenDelimiter($delimiter);
        }

        $this->tokenEnd = $delimiter;
    }
636377
public function setContentType($contentType, $charset=NULL){
        $this->setHeader("Content-Type", $contentType);
        if($charset != NULL){
            $this->setHeader("charset", $charset);
        }
    }
552204
public function setToken(string $token)
    {
        if (empty($selector)) {
            Throw new TokenException('Empty tokens are not allowed.');
        }

        $this->token = $token;
    }
594517
public static function header(string $content, string $type = null)
    {
        if ($type) {
            header($content.': '.$type.'; charset=UTF-8');
        } else {
            header($content);
        }
    }