Parses the tokens (passed in reference!) and returns a TypeDescriptor for the parsed tokens.


@param array $tokens
@throws MoufTypeParserException
@return TypeDescriptor
429203
public static function parseTokens(&$tokens) {
        do {
		    $typeArray = array_shift($tokens);
            if ($typeArray['token'] == "T_WHITESPACE"){
                continue;
            }
            else {
                break;
            }
        }while(true);

        if ($typeArray['token'] != 'T_TYPE') {
			throw new MoufTypeParserException("Invalid type! Expecting a type name. Got '".$typeArray['match']."'");
		}
		
		$type = new TypeDescriptor();
		$type->type = $typeArray['match'];
		
		if (empty($typeArray)) {
			return $type;
		}
		
		do {
			$continue = false;
			$nextToken = array_shift($tokens);
		
			switch ($nextToken['token']) {
				case 'T_ARRAY';
					$continue = true;
					$subType = $type;
					$type = new TypeDescriptor();
					$type->type = 'array';
					$type->subType = $subType;
					break;
				case 'T_END_ARRAY';
                    array_unshift($tokens, $nextToken);
					break;
				case 'T_START_ARRAY';
					$tok1 = self::getNthTokenWithoutWhitespace($tokens, 1);
					if ($tok1['token'] == 'T_COMA') {
						$tok0 = self::getNthTokenWithoutWhitespace($tokens, 0);
						if ($tok0['token'] != 'T_TYPE') {
							throw new MoufTypeParserException("Invalid type! Expecting a type name. Got ".$tokens['match']);
						}
						$type->keyType = $tok0['match'];
						// Let's shift twice, without counting whitespaces:
						while ($tokens[0] != $tok1) {
							array_shift($tokens);
						}
						array_shift($tokens);
						// TODO: with a state machine, we could handle whitespaces in a best way
						// TODO: develop a statemachine!
						while ($tokens[0]['token'] == 'T_WHITESPACE') {
							array_shift($tokens);
						}
						
					}
					$type->subType = TypeDescriptor::parseTokens($tokens);

                    while ($tokens[0]['token'] == 'T_WHITESPACE') {
                        array_shift($tokens);
                    }

                    if ($tokens[0]['token'] != 'T_END_ARRAY'){
                        throw new MoufTypeParserException("Invalid type! Expecting >. Got '".$tokens[0]['match']."'");
                    }

                    array_shift($tokens);

					break;
				case 'T_OR':
					break;
			}
		} while ($continue);
		
		return $type;
	}
1002602
protected function parseCreateIndex(SQLTokenIterator $tokens)
    {

        /* @var $entity CreateIndexStatement */
        $entity = new CreateIndexStatement();

        ### FLAGS

        if ($tokens->isToken(SqlToken::T_UNIQUE(), TokenIterator::PREVIOUS, [SqlToken::T_PRIMARY()])) {
            $entity->setIsUnique(true);
        }

        if ($tokens->isToken(SqlToken::T_PRIMARY(), TokenIterator::PREVIOUS, [SqlToken::T_UNIQUE()])) {
            $entity->setIsPrimary(true);
        }

        ### NAME

        if (!$tokens->seekTokenNum(T_STRING)) {
            throw new MalformedSqlException("Missing valid index-name!", $tokens);
        }

        $entity->setName($tokens->getCurrentTokenString());

        ### USING

        if ($tokens->seekTokenNum(SqlToken::T_USING())) {
            switch(true){

                // TODO: R-TREE index not implemented yet!
                case $tokens->seekTokenNum(SqlToken::T_RTREE()):
                case $tokens->seekTokenNum(SqlToken::T_BTREE()):
                    $entity->setIndexType(IndexType::BTREE());
                    break;

                case $tokens->seekTokenNum(SqlToken::T_HASH()):
                    $entity->setIndexType(IndexType::HASH());
                    break;

                default:
                    throw new MalformedSqlException("Invalid index-type specified!", $tokens);
            }
        }

        ### TABLE

        if (!$tokens->seekTokenNum(SqlToken::T_ON())) {
            throw new MalformedSqlException("Missing T_ON for CREATE INDEX statement!", $tokens);
        }

        if (!$this->tableParser->canParseTokens($tokens)) {
            throw new MalformedSqlException("Missing valid table-specifier for CREATE INDEX statement!", $tokens);
        }

        $entity->setTable($this->tableParser->convertSqlToJob($tokens));

        ### COLUMNS

        if (!$tokens->seekTokenText('(')) {
            throw new MalformedSqlException("Missing beginning parenthesis holding columns in CREATE INDEX statement!", $tokens);
        }

        do {
            if (!$this->columnParser->canParseTokens($tokens)) {
                throw new MalformedSqlException("Missing valid column-specifier in CREATE INDEX statement!", $tokens);
            }

            $column = $this->columnParser->convertSqlToJob($tokens);

            $length = null;
            if ($tokens->seekTokenText('(')) {
                if (!$this->valueParser->canParseTokens($tokens)) {
                    throw new MalformedSqlException("Missing valid column-length in CREATE INDEX statement!", $tokens);
                }

                $length = $this->valueParser->convertSqlToJob($tokens);

                if (!$tokens->seekTokenText(')')) {
                    throw new MalformedSqlException("Missing closing parenthesis holding column-length in CREATE INDEX statement!", $tokens);
                }
            }

            $direction = null;
            if ($tokens->seekTokenNum(SqlToken::T_ASC())) {
                $direction = SqlToken::T_ASC();
            } elseif ($tokens->seekTokenNum(SqlToken::T_DESC())) {
                $direction = SqlToken::T_DESC();
            }

            $entity->addColumn($column, $length, $direction);

        } while ($tokens->seekTokenText(','));

        if (!$tokens->seekTokenText(')')) {
            throw new MalformedSqlException("Missing closing parenthesis holding columns in CREATE INDEX statement!", $tokens);
        }

        ### WITH PARSER

        if ($tokens->seekTokenNum(SqlToken::T_WITH())) {
            if (!$tokens->seekTokenNum(SqlToken::T_PARSER())) {
                throw new MalformedSqlException("Missing T_PARSER after T_WITH in CREATE INDEX statement!", $tokens);
            }

            if (!$tokens->seekTokenNum(T_STRING)) {
                throw new MalformedSqlException("Missing valid parser name after WITH PARSER in CREATE INDEX statement!", $tokens);
            }

            $entity->setParser($tokens->getCurrentTokenString());
        }

        return $entity;
    }
1060705
private function parse($tokens, ?TypeContext $context = null): array
    {
        $count = count($tokens);
        $tagCount = 0;
        $tags = [];

        for ($i = 1; $i < $count; $i += 2) {
            $tags[] = $this->tagFactory->create($tokens[$i], $context);
            $tokens[$i] = '%' . ++$tagCount . '$s';
        }

        //In order to allow "literal" inline tags, the otherwise invalid
        //sequence "{@}" is changed to "@", and "{}" is changed to "}".
        //"%" is escaped to "%%" because of vsprintf.
        //See unit tests for examples.
        for ($i = 0; $i < $count; $i += 2) {
            $tokens[$i] = str_replace(['{@}', '{}', '%'], ['@', '}', '%%'], $tokens[$i]);
        }

        return [implode('', $tokens), $tags];
    }
416507
protected function parseTokens(array $tokens)
    {
        //Current active token
        $activeToken = [];

        //Some blocks can be named as parent. We have to make sure we closing the correct one
        $activeLevel = 0;

        //Content to represent full tag declaration (including body)
        $activeContent = [];

        foreach ($tokens as $token) {
            $tokenType = $token[HtmlTokenizer::TOKEN_TYPE];

            if (empty($activeToken)) {
                switch ($tokenType) {
                    case HtmlTokenizer::TAG_VOID:
                    case HtmlTokenizer::TAG_SHORT:
                        //Token will be clarified using Supervisor
                        $this->mountToken($token);
                        break;

                    case HtmlTokenizer::TAG_OPEN:
                        $activeToken = $token;
                        break;

                    case HtmlTokenizer::TAG_CLOSE:
                        if ($this->compiler->getSyntax()->isStrict()) {
                            throw new StrictSyntaxException(
                                "Unpaired close tag '{$token[HtmlTokenizer::TOKEN_NAME]}'.",
                                $token
                            );
                        }
                        break;
                    case HtmlTokenizer::PLAIN_TEXT:
                        //Everything outside any tag
                        $this->mountContent([$token]);
                        break;
                }

                continue;
            }

            if (
                $tokenType != HtmlTokenizer::PLAIN_TEXT
                && $token[HtmlTokenizer::TOKEN_NAME] == $activeToken[HtmlTokenizer::TOKEN_NAME]
            ) {
                if ($tokenType == HtmlTokenizer::TAG_OPEN) {
                    $activeContent[] = $token;
                    $activeLevel++;
                } elseif ($tokenType == HtmlTokenizer::TAG_CLOSE) {
                    if ($activeLevel === 0) {
                        //Closing current token
                        $this->mountToken($activeToken, $activeContent, $token);
                        $activeToken = $activeContent = [];
                    } else {
                        $activeContent[] = $token;
                        $activeLevel--;
                    }
                } else {
                    //Short tag with same name (used to call for parent content)s
                    $activeContent[] = $token;
                }

                continue;
            }

            //Collecting token content
            $activeContent[] = $token;
        }

        //Everything after last tag
        $this->mountContent($activeContent);
    }
809493
protected function expectTokenType($type)
    {
        $types = (array) $type;
        if (in_array($this->predictTokenType(), $types)) {
            return $this->lexer->getAdvancedToken();
        }

        $token = $this->lexer->predictToken();

        throw new ParserException(sprintf(
            'Expected %s token, but got %s on line: %d%s',
            implode(' or ', $types),
            $this->predictTokenType(),
            $token['line'],
            $this->file ? ' in file: ' . $this->file : ''
        ));
    }
1037551
protected function separateActiveAndInactiveTokens(array $tokens)
    {
        if ($this->request === null) {
            return;
        }

        /** @var $token TokenInterface */
        foreach ($tokens as $token) {
            if ($this->isTokenActive($token)) {
                $this->activeTokens[$token->getAuthenticationProviderName()] = $token;
            } else {
                $this->inactiveTokens[$token->getAuthenticationProviderName()] = $token;
            }
        }
    }
1202932
private function sortTokens(array $tokens): array
    {
        $tagCounts = $this->docTagHelper->getTagCounts($tokens);

        usort($tokens, function (array $leftToken, array $rightToken) use ($tagCounts): int {
            return $this->compareTokensForSorting($leftToken, $rightToken, $tagCounts);
        });

        return $tokens;
    }
431597
private function getCurrentConstantKeyFromClassTokens($tokens)
    {
        $parsingStateConstDeclarationFound = false;
        $constDeclarationKey = 0;

        foreach ($tokens as $key => $token) {
            if (is_array($token) === true && $token[0] === T_CONST) {
                $parsingStateConstDeclarationFound = true;
                $constDeclarationKey = $key;
            }

            if ($parsingStateConstDeclarationFound === true && $token[0] === T_STRING) {
                if ($token[1] === $this->getName()) {
                    break;
                } else {
                    $parsingStateConstDeclarationFound = false;
                }
            }
        }

        return $constDeclarationKey;
    }
275806
static function parse(array $tokens, &$i) {
    if (T_NS_SEPARATOR !== $tokens[$i][0]) {
      return FALSE;
    }
    ++$i;
    $qcnString = '';
    while (TRUE) {
      if (T_STRING !== $tokens[$i][0]) {
        return FALSE;
      }
      $qcnString .= $tokens[$i][1];
      ++$i;
      if (T_NS_SEPARATOR !== $tokens[$i][0]) {
        break;
      }
      $qcnString .= '\\';
      ++$i;
    }
    return Fqcn::createFromValidFqcnString($qcnString);
  }
1237904
protected function __add( array $tokens )
	{
		$updated = false;

		foreach ( $tokens as $token ) {
			if ( ! $this->__contains( $token ) ) {
				$this->tokens[] = $token;
				$updated = true;
			}
		}

		if ( $updated ) {
			$this->__update();
		}

		return $this;
	}
546583
public function process($tokens) {
        $prev_category = "";
        $token_category = "";
        $skip_next = 0;
        $out = false;

	// $tokens may come as a numeric indexed array starting with an index greater than 0 (or as a boolean)
	$tokenCount = count($tokens);
        if ( is_array($tokens) ){
          $tokens = array_values($tokens);
        }
        for ($tokenNumber = 0; $tokenNumber < $tokenCount; ++$tokenNumber) {

            // https://github.com/greenlion/PHP-SQL-Parser/issues/279
            // https://github.com/sinri/PHP-SQL-Parser/commit/eac592a0e19f1df6f420af3777a6d5504837faa7
            // as there is no pull request for 279 by the user. His solution works and tested.
            if (!isset($tokens[$tokenNumber])) continue;// as a fix by Sinri 20180528
            $token = $tokens[$tokenNumber];
            $trim = trim($token); // this removes also \n and \t!

            // if it starts with an "(", it should follow a SELECT
            if ($trim !== "" && $trim[0] === "(" && $token_category === "") {
                $token_category = 'BRACKET';
                $prev_category = $token_category;
            }

            /*
             * If it isn't obvious, when $skip_next is set, then we ignore the next real token, that is we ignore whitespace.
             */
            if ($skip_next > 0) {
                if ($trim === "") {
                    if ($token_category !== "") { // is this correct??
                        $out[$token_category][] = $token;
                    }
                    continue;
                }
                // to skip the token we replace it with whitespace
                $trim = "";
                $token = "";
                $skip_next--;
                if ($skip_next > 0) {
                    continue;
                }
            }

            $upper = strtoupper($trim);
            switch ($upper) {

            /* Tokens that get their own sections. These keywords have subclauses. */
            case 'SELECT':
            case 'ORDER':
            case 'VALUES':
            case 'GROUP':
            case 'HAVING':
            case 'WHERE':
            case 'CALL':
            case 'PROCEDURE':
            case 'FUNCTION':
            case 'SERVER':
            case 'LOGFILE':
            case 'DEFINER':
            case 'RETURNS':
            case 'TABLESPACE':
            case 'TRIGGER':
            case 'DO':
            case 'FLUSH':
            case 'KILL':
            case 'RESET':
            case 'STOP':
            case 'PURGE':
            case 'EXECUTE':
            case 'PREPARE':
                $token_category = $upper;
                break;

            case 'DEALLOCATE':
                if ($trim === 'DEALLOCATE') {
                    $skip_next = 1;
                }
                $token_category = $upper;
                break;

            case 'DUPLICATE':
                if ($token_category !== 'VALUES') {
                    $token_category = $upper;
                }
                break;

            case 'SET':
                if ($token_category !== 'TABLE') {
                    $token_category = $upper;
                }
                break;

            case 'LIMIT':
            case 'PLUGIN':
            // no separate section
                if ($token_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'FROM':
            // this FROM is different from FROM in other DML (not join related)
                if ($token_category === 'PREPARE') {
                    continue 2;
                }
                // no separate section
                if ($token_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'EXPLAIN':
            case 'DESCRIBE':
            case 'SHOW':
                $token_category = $upper;
                break;

            case 'DESC':
                if ($token_category === '') {
                    // short version of DESCRIBE
                    $token_category = $upper;
                }
                // else direction of ORDER-BY
                break;

            case 'RENAME':
                $token_category = $upper;
                break;

            case 'DATABASE':
            case 'SCHEMA':
                if ($prev_category === 'DROP') {
                    continue;
                }
                if ($prev_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'EVENT':
            // issue 71
                if ($prev_category === 'DROP' || $prev_category === 'ALTER' || $prev_category === 'CREATE') {
                    $token_category = $upper;
                }
                break;

            case 'DATA':
            // prevent wrong handling of DATA as keyword
                if ($prev_category === 'LOAD') {
                    $token_category = $upper;
                }
                break;

            case 'INTO':
            // prevent wrong handling of CACHE within LOAD INDEX INTO CACHE...
                if ($prev_category === 'LOAD') {
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                $token_category = $prev_category = $upper;
                break;

            case 'USER':
            // prevent wrong processing as keyword
                if ($prev_category === 'CREATE' || $prev_category === 'RENAME' || $prev_category === 'DROP') {
                    $token_category = $upper;
                }
                break;

            case 'VIEW':
            // prevent wrong processing as keyword
                if ($prev_category === 'CREATE' || $prev_category === 'ALTER' || $prev_category === 'DROP') {
                    $token_category = $upper;
                }
                break;

            /*
             * These tokens get their own section, but have no subclauses. These tokens identify the statement but have no specific subclauses of their own.
             */
            case 'DELETE':
            case 'ALTER':
            case 'INSERT':
            case 'TRUNCATE':
            case 'OPTIMIZE':
            case 'GRANT':
            case 'REVOKE':
            case 'HANDLER':
            case 'LOAD':
            case 'ROLLBACK':
            case 'SAVEPOINT':
            case 'UNLOCK':
            case 'INSTALL':
            case 'UNINSTALL':
            case 'ANALZYE':
            case 'BACKUP':
            case 'CHECKSUM':
            case 'REPAIR':
            case 'RESTORE':
            case 'HELP':
                $token_category = $upper;
                // set the category in case these get subclauses in a future version of MySQL
                $out[$upper][0] = $trim;
                continue 2;

            case 'REPLACE':
            	if ($prev_category === '') {
            		// set the category in case these get subclauses in a future version of MySQL
            		$token_category = $upper;
            		$out[$upper][0] = $trim;
            		continue 2;
            	}
                // part of the CREATE TABLE statement or a function
                $out[$prev_category][] = $trim;
                continue 2;

            case 'IGNORE':
                if ($prev_category === 'TABLE') {
                    // part of the CREATE TABLE statement
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                if ($token_category === 'FROM') {
                    // part of the FROM statement (index hint)
                    $out[$token_category][] = $trim;
                    continue 2;
                }
                $out['OPTIONS'][] = $upper;
                continue 2;

            case 'CHECK':
                if ($prev_category === 'TABLE') {
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                $token_category = $upper;
                $out[$upper][0] = $trim;
                continue 2;

            case 'CREATE':
                if ($prev_category === 'SHOW') {
                    continue;
                }
                $token_category = $upper;
                break;

            case 'INDEX':
	            if ( in_array( $prev_category, array( 'CREATE', 'DROP' ) ) ) {
		            $out[ $prev_category ][] = $trim;
		            $token_category          = $upper;
	            }
	            break;

            case 'TABLE':
                if ($prev_category === 'CREATE') {
                    $out[$prev_category][] = $trim;
                    $token_category = $upper;
                }
                if ($prev_category === 'TRUNCATE') {
                    $out[$prev_category][] = $trim;
                    $token_category = $upper;
                }
                break;

            case 'TEMPORARY':
                if ($prev_category === 'CREATE') {
                    $out[$prev_category][] = $trim;
                    $token_category = $prev_category;
                    continue 2;
                }
                break;

            case 'IF':
                if ($prev_category === 'TABLE') {
                    $token_category = 'CREATE';
                    $out[$token_category] = array_merge($out[$token_category], $out[$prev_category]);
                    $out[$prev_category] = array();
                    $out[$token_category][] = $trim;
                    $prev_category = $token_category;
                    continue 2;
                }
                break;

            case 'NOT':
                if ($prev_category === 'CREATE') {
                    $token_category = $prev_category;
                    $out[$prev_category][] = $trim;
                    continue 2;
                }
                break;

            case 'EXISTS':
                if ($prev_category === 'CREATE') {
                    $out[$prev_category][] = $trim;
                    $prev_category = $token_category = 'TABLE';
                    continue 2;
                }
                break;

            case 'CACHE':
                if ($prev_category === "" || $prev_category === 'RESET' || $prev_category === 'FLUSH'
                    || $prev_category === 'LOAD') {
                    $token_category = $upper;
                    continue 2;
                }
                break;

            /* This is either LOCK TABLES or SELECT ... LOCK IN SHARE MODE */
            case 'LOCK':
                if ($token_category === "") {
                    $token_category = $upper;
                    $out[$upper][0] = $trim;
                } elseif ($token_category === 'INDEX') {
                    break;
                } else {
                    $trim = 'LOCK IN SHARE MODE';
                    $skip_next = 3;
                    $out['OPTIONS'][] = $trim;
                }
                continue 2;

            case 'USING': /* USING in FROM clause is different from USING w/ prepared statement*/
                if ($token_category === 'EXECUTE') {
                    $token_category = $upper;
                    continue 2;
                }
                if ($token_category === 'FROM' && !empty($out['DELETE'])) {
                    $token_category = $upper;
                    continue 2;
                }
                break;

            /* DROP TABLE is different from ALTER TABLE DROP ... */
            case 'DROP':
                if ($token_category !== 'ALTER') {
                    $token_category = $upper;
                    continue 2;
                }
                break;

            case 'FOR':
                if ($prev_category === 'SHOW') {
                    continue;
                }
                $skip_next = 1;
                $out['OPTIONS'][] = 'FOR UPDATE'; // TODO: this could be generate problems within the position calculator
                continue 2;

            case 'UPDATE':
                if ($token_category === "") {
                    $token_category = $upper;
                    continue 2;
                }
                if ($token_category === 'DUPLICATE') {
                    continue 2;
                }
                break;

            case 'START':
                $trim = "BEGIN";
                $out[$upper][0] = $upper; // TODO: this could be generate problems within the position calculator
                $skip_next = 1;
                break;

            // This token is ignored, except within RENAME
            case 'TO':
                if ($token_category === 'RENAME') {
                    break;
                }
                continue 2;

            // This token is ignored, except within CREATE TABLE
            case 'BY':
                if ($prev_category === 'TABLE') {
                    break;
                }
                continue 2;

            // These tokens are ignored.
            case 'ALL':
            case 'SHARE':
            case 'MODE':
            case ';':
                continue 2;

            case 'KEY':
                if ($token_category === 'DUPLICATE') {
                    continue 2;
                }
                break;

            /* These tokens set particular options for the statement. */
            case 'LOW_PRIORITY':
            case 'DELAYED':
            case 'QUICK':
            case 'HIGH_PRIORITY':
                $out['OPTIONS'][] = $trim;
                continue 2;

            case 'USE':
                if ($token_category === 'FROM') {
                    // index hint within FROM clause
                    $out[$token_category][] = $trim;
                    continue 2;
                }
                // set the category in case these get subclauses in a future version of MySQL
                $token_category = $upper;
                $out[$upper][0] = $trim;
                continue 2;

            case 'FORCE':
                if ($token_category === 'FROM') {
                    // index hint within FROM clause
                    $out[$token_category][] = $trim;
                    continue 2;
                }
                $out['OPTIONS'][] = $trim;
                continue 2;

            case 'WITH':
                if ($token_category === 'GROUP') {
                    $skip_next = 1;
                    $out['OPTIONS'][] = 'WITH ROLLUP'; // TODO: this could be generate problems within the position calculator
                    continue 2;
                }
                if ($token_category === '') {
                	$token_category = $upper;
                }
                break;

            case 'AS':
                break;

            case '':
            case ',':
            case ';':
                break;

            default:
                break;
            }

            // remove obsolete category after union (empty category because of
            // empty token before select)
            if ($token_category !== "" && ($prev_category === $token_category)) {
                $out[$token_category][] = $token;
            }

            $prev_category = $token_category;
        }

        return parent::process($out);
    }
794867
public static function create(array $tokens) {
        HashMap::assert($tokens, 'tokens');

        if (empty($tokens['ORDER'])) return [];

        $headers = [];
        $orderQueryArr = array_map(function($order){
            $query = end($order['no_quotes']['parts']);
            if (empty($query)) return null;
            return !isset($order['direction']) ? $query : $query . ' ' . $order['direction'];
        },$tokens['ORDER']);
        array_push($headers, 'Order: '.implode(',',$orderQueryArr));
        return $headers;
    }
498338
protected function processRawContentTagTokens(array $tokens) : array
    {
        $rawTagName = null;
        return array_reduce($tokens, function (array $tokens, $token) use (&$rawTagName) {
            if (!isset($rawTagName)) {
                // If not in a raw-content tag and current tag is a raw-content, store a tag name.
                if ($token instanceof TagToken) {
                    $tagName = $token->getTagName();
                    $tagDef = $this->tagDefs->get($tagName);
                    if ($tagDef->rawContent()) {
                        $rawTagName = $tagName;
                    }
                }
            } elseif ($token instanceof TagToken && $token->getTagName() === $rawTagName) {
                // If in a raw-content tag and current tag is closing for it, unset the stored tag name.
                $rawTagName = null;
            } else {
                // If in a raw-content tag, return current token as a text token.
                $text = $token->getRawText();
                $token = new TextToken($text);
            }

            // Append current token.
            return array_merge($tokens, [$token]);
        }, []);
    }
1235605
private function parseTypes(\ArrayIterator $tokens, Context $context, $parserContext)
    {
        $types = [];
        $token = '';
        while ($tokens->valid()) {
            $token = $tokens->current();

            if ($token === '|') {
                if (count($types) === 0) {
                    throw new \RuntimeException(
                        'A type is missing before a type separator'
                    );
                }

                if ($parserContext !== self::PARSER_IN_COMPOUND
                    && $parserContext !== self::PARSER_IN_ARRAY_EXPRESSION
                    && $parserContext !== self::PARSER_IN_COLLECTION_EXPRESSION
                ) {
                    throw new \RuntimeException(
                        'Unexpected type separator'
                    );
                }

                $tokens->next();
            } elseif ($token === '?') {
                if ($parserContext !== self::PARSER_IN_COMPOUND
                    && $parserContext !== self::PARSER_IN_ARRAY_EXPRESSION
                    && $parserContext !== self::PARSER_IN_COLLECTION_EXPRESSION
                ) {
                    throw new \RuntimeException(
                        'Unexpected nullable character'
                    );
                }

                $tokens->next();
                $type = $this->parseTypes($tokens, $context, self::PARSER_IN_NULLABLE);
                $types[] = new Nullable($type);
            } elseif ($token === '(') {
                $tokens->next();
                $type = $this->parseTypes($tokens, $context, self::PARSER_IN_ARRAY_EXPRESSION);

                $resolvedType = new Array_($type);

                $token = $tokens->current();
                // Someone did not properly close their array expression ..
                if ($token === null) {
                    break;
                }

                // we generate arrays corresponding to the number of '[]' after the ')'
                $numberOfArrays = (strlen($token) - 1) / 2;
                for ($i = 0; $i < $numberOfArrays - 1; ++$i) {
                    $resolvedType = new Array_($resolvedType);
                }

                $types[] = $resolvedType;
                $tokens->next();
            } elseif ($parserContext === self::PARSER_IN_ARRAY_EXPRESSION && $token[0] === ')') {
                break;
            } elseif ($token === '<') {
                if (count($types) === 0) {
                    throw new \RuntimeException(
                        'Unexpected collection operator "<", class name is missing'
                    );
                }

                $classType = array_pop($types);
                if ($classType !== null) {
                    $types[] = $this->resolveCollection($tokens, $classType, $context);
                }

                $tokens->next();
            } elseif ($parserContext === self::PARSER_IN_COLLECTION_EXPRESSION
                && ($token === '>' || trim($token) === ',')
                ) {
                break;
            } else {
                $type = $this->resolveSingleType($token, $context);
                $tokens->next();
                if ($parserContext === self::PARSER_IN_NULLABLE) {
                    return $type;
                }

                $types[] = $type;
            }
        }

        if ($token === '|') {
            throw new \RuntimeException(
                'A type is missing after a type separator'
            );
        }

        if (count($types) === 0) {
            if ($parserContext === self::PARSER_IN_NULLABLE) {
                throw new \RuntimeException(
                    'A type is missing after a nullable character'
                );
            }

            if ($parserContext === self::PARSER_IN_ARRAY_EXPRESSION) {
                throw new \RuntimeException(
                    'A type is missing in an array expression'
                );
            }

            if ($parserContext === self::PARSER_IN_COLLECTION_EXPRESSION) {
                throw new \RuntimeException(
                    'A type is missing in a collection expression'
                );
            }
        } elseif (count($types) === 1) {
            return $types[0];
        }

        return new Compound($types);
    }
297835
public Parse parse(Parse tokens) {
    Parse p = parse(tokens,1)[0];
    setParents(p);
    return p;
  }
958891
public function translate(array $tokens)
    {
        $this->operatorStack = new SplStack();
        $this->outputQueue = new SplQueue();
        foreach($tokens as $token) {
            switch ($token->getType()) {
                case Token::T_OPERAND:
                    $this->outputQueue->enqueue($token);
                    break;
                case Token::T_OPERATOR:
                    $o1 = $token;
                    while($this->hasOperatorInStack()
                            && ($o2 = $this->operatorStack->top())
                            && $o1->hasLowerPriority($o2))
                    {
                        $this->outputQueue->enqueue($this->operatorStack->pop());
                    }

                    $this->operatorStack->push($o1);
                    break;
                case Token::T_LEFT_BRACKET:
                    $this->operatorStack->push($token);
                    break;
                case Token::T_RIGHT_BRACKET:
                    while((!$this->operatorStack->isEmpty()) && (Token::T_LEFT_BRACKET != $this->operatorStack->top()->getType())) {
                        $this->outputQueue->enqueue($this->operatorStack->pop());
                    }
                    if($this->operatorStack->isEmpty()) {
                        throw new InvalidArgumentException(sprintf('Mismatched parentheses: %s', implode(' ',$tokens)));
                    }
                    $this->operatorStack->pop();
                    break;
                default:
                    throw new InvalidArgumentException(sprintf('Invalid token detected: %s', $token));
                    break;
            }
        }
        while($this->hasOperatorInStack()) {
            $this->outputQueue->enqueue($this->operatorStack->pop());
        }

        if(!$this->operatorStack->isEmpty()) {
            throw new InvalidArgumentException(sprintf('Mismatched parenthesis or misplaced number: %s', implode(' ',$tokens)));
        }

        return iterator_to_array($this->outputQueue);
    }
712975
private function parseNamespace($tokens)
    {
        $namespace = null;
        $tokenCount = count($tokens);
        for ($offset=0; $offset < $tokenCount; $offset++) {
            if (!is_array($tokens[$offset])) {
                continue;
            }

            if (T_NAMESPACE === $tokens[$offset][0]) {
                $offset++; // the next token is a whitespace

                return $this->parseNamespaceString($tokens, $offset);
            }
        }

        return $namespace;
    }