Returns the tokens for the next "plain" value.

@return array The tokens.

@throws Exception
@throws SyntaxException If a syntax error is found.
182129
private function getPlainValue()
    {
        // check if array, then hand off
        if ($this->lexer->isNextToken(DocLexer::T_OPEN_CURLY_BRACES)) {
            return $this->getArray();
        }

        // check if nested annotation, then hand off
        if ($this->lexer->isNextToken(DocLexer::T_AT)) {
            return $this->getAnnotation();
        }

        // check if constant, then hand off
        if ($this->lexer->isNextToken(DocLexer::T_IDENTIFIER)) {
            return $this->getConstant();
        }

        $tokens = array();

        // determine type, or throw syntax error if unrecognized
        switch ($this->lexer->lookahead['type']) {
            case DocLexer::T_FALSE:
            case DocLexer::T_FLOAT:
            case DocLexer::T_INTEGER:
            case DocLexer::T_NULL:
            case DocLexer::T_STRING:
            case DocLexer::T_TRUE:
                $this->match($this->lexer->lookahead['type']);

                $tokens[] = array(
                    $this->lexer->token['type'],
                    $this->lexer->token['value']
                );

                break;
            default:
                throw SyntaxException::expectedToken(
                    'PlainValue',
                    null,
                    $this->lexer
                );
        }

        return $tokens;
    }
1212363
public function next() {
    $this->position++;
    if ($this->position >= $this->length) {
      $this->position = $this->length;
      return NULL;
    }
    return $this->tokens[$this->position];
  }
768319
public List<E> tokenize() {
    // System.out.println("tokenize called");
    List<E> result = new ArrayList<E>();
    while (hasNext()) {
      result.add(next());
    }
    return result;
  }
717486
public function getPreparedValues()
    {
        $preparedValues = [];
        foreach ($this->getAll() as $name => $values) {
            $preparedValues = array_merge($preparedValues, $this->prepareValues($name, $values));
        }

        return $preparedValues;
    }
262080
protected function parseArguments() {
    $token_name = $this->getToken();
    $delimiter = '/';
    $matches = array();
    if (preg_match($delimiter . preg_quote($token_name) . '<(.+)>.*' . $delimiter, $this->path, $matches)) {
      // Some arguments were found.
      return explode(',', $matches[1]);
    }
    return array();
  }
1070843
public String[] getTokenArray() {
        checkTokenized();
        if (this.tokens == null) {
            return null;
        }
        final String[] clone = new String[this.tokens.length];
        for (int i = 0; i < this.tokens.length; i++) {
            clone[i] = this.tokens[i];
        }        
        return clone;
    }
164586
public function getStaticTokens()
    {
        $tokens = [];
        $sites = [];
        if (!$this->getValid()) {
            return $sites;
        }
        foreach ($this->xml->children() as $child) {
            if ($child->getName() == "token") {
                $token = $this->parseToken($child);
                if ($token !== false) {
                    $tokens[$token['token']] = $token;
                }
            }
        }
        return $tokens;
    }
1069322
public function readToken(): Token
    {
        if ($this->getBuffer()->isEnd()) {
            try {
                $token = $this
                    ->getScanner()
                    ->readToken();
            } catch (UnknownSyntaxException $e) {
                throw new UnknownSyntaxException(
                    "Unknown syntax error near position #{$this->getNextPosition()}",
                    $this->getNextPosition(),
                    $e
                );
            }
            $this->assertErrorToken($token);
            $this
                ->getBuffer()
                ->addToken($token);
        }
        return $this
            ->getBuffer()
            ->readToken();
    }
650801
protected function next()
    {
        while ($token = array_shift($this->tokens)) {
            $this->line += substr_count($this->value($token), "\n");

            if (is_array($token) && in_array($token[0], [T_WHITESPACE, T_COMMENT, T_DOC_COMMENT], true)) {
                continue;
            }

            return $token;
        }
    }
1241854
private function parseToken()
    {
        $token = '';

        while ($this->valid()) {
            if (ctype_space($this->current)) {
                // Skip space
                $this->next();

                break;
            }

            if ('\\' === $this->current) {
                $token .= $this->parseEscapeSequence();
            } elseif ("'" === $this->current || '"' === $this->current) {
                $token .= $this->parseQuotedString();
            } else {
                $token .= $this->current;
                $this->next();
            }
        }

        return $token;
    }
120661
protected function lexEnumValuesDefinition(): array
    {
        return $this->peek(TokenKindEnum::BRACE_L)
            ? $this->many(
                TokenKindEnum::BRACE_L,
                [$this, 'lexEnumValueDefinition'],
                TokenKindEnum::BRACE_R
            )
            : [];
    }
247677
protected function _parseArgs()
    {
        $cpt = 0;
        $value = $name = '';
        $args = [];
        $body = '';
        while ($token = $this->_stream->current(true)) {
            $body .= $token[1];
            switch ($token[0]) {
                case '(':
                    if ($cpt) {
                        $value .= $token[1];
                    }
                    $cpt++;
                    break;
                case '=':
                    $name = $value;
                    $value = '';
                    break;
                case ')':
                    $cpt--;
                    if ($cpt) {
                        $value .= $token[1];
                        break;
                    }
                    /* Same behavior as comma */
                case ',':
                    $value = trim($value);
                    if ($value !== '') {
                        $name ? $args[trim($name)] = $value : $args[] = $value;
                    }
                    $name = $value = '';
                    break;
                default:
                    $value .= $token[1];
                    break;
            }
            if ($token[1] === ')' && $cpt === 0) {
                break;
            }
            $this->_stream->next();
        }
        return compact('args', 'body');
    }
1332476
public function getNextToken()
    {
        $token = \array_pop($this->stack);
        if ($token !== null) {
            return $token;
        }

        if (($byte = $this->streamReader->readByte()) === false) {
            return false;
        }

        if ($byte === "\x20" ||
            $byte === "\x0A" ||
            $byte === "\x0D" ||
            $byte === "\x0C" ||
            $byte === "\x09" ||
            $byte === "\x00"
        ) {
            if ($this->leapWhiteSpaces() === false) {
                return false;
            }
            $byte = $this->streamReader->readByte();
        }

        switch ($byte) {
            case '/':
            case '[':
            case ']':
            case '(':
            case ')':
            case '{':
            case '}':
            case '<':
            case '>':
                return $byte;
            case '%':
                $this->streamReader->readLine();
                return $this->getNextToken();
        }

        /* This way is faster than checking single bytes.
         */
        $bufferOffset = $this->streamReader->getOffset();
        do {
            $lastBuffer = $this->streamReader->getBuffer(false);
            $pos = \strcspn(
                $lastBuffer,
                "\x00\x09\x0A\x0C\x0D\x20()<>[]{}/%",
                $bufferOffset
            );
        } while (
            // Break the loop if a delimiter or white space char is matched
            // in the current buffer or increase the buffers length
            $lastBuffer !== false &&
            (
                $bufferOffset + $pos === \strlen($lastBuffer) &&
                $this->streamReader->increaseLength()
            )
        );

        $result = \substr($lastBuffer, $bufferOffset - 1, $pos + 1);
        $this->streamReader->setOffset($bufferOffset + $pos);

        return $result;
    }
1105189
public function getTokens()
    {
        preg_match_all("/(\:\w+)/", $this->getSchema(), $matches);
        $tokens = array();

        foreach ($matches[0] as $match) {
            $tokens[$match] = array();
        }

        return $tokens;
    }
862254
protected function _decodeArray()
    {
        $result = array();
        $tok = $this->_getNextToken(); // Move past the '['
        $index  = 0;

        while ($tok && $tok != self::RBRACKET) {
            $result[$index++] = $this->_decodeValue();

            $tok = $this->token;

            if ($tok == self::RBRACKET || !$tok) {
                break;
            }

            if ($tok != self::COMMA) {
                throw new RuntimeException('Missing "," in array encoding: ' . $this->source);
            }

            $tok = $this->_getNextToken();
        }

        $this->_getNextToken();
        return $result;
    }
184336
function parse()
    {
        $criteria        = $this->criteria;
        $expectAsLiteral = $this->expectAsLiteral;


        $TOKENS  = preg_quote(static::TOKENS);

        $expectAsLiteral = (empty($expectAsLiteral)) ? '/@+-.' : $expectAsLiteral;
        $LITERAL = preg_quote($expectAsLiteral).'A-Za-z0-9_';


        $currentPos = 0;
        $length     = strlen($criteria);
        $parts      = [];
        $levelParts = [&$parts];
        $level      = 0;

        while ($currentPos < $length)
        {
            preg_match("(\G(?P<_literal_>[$LITERAL]*)(?P<_token_>[$TOKENS]|$))"
                , $criteria
                , $matches
                , 0
                , $currentPos
            );

            if ( empty($matches) )
                break;

            if (! empty($matches['_literal_']) ) {
                $levelParts[$level][] = ['_literal_' => $matches['_literal_']];
                $currentPos += strlen($matches['_literal_']);
            }

            # Deal With Token:
            if ( !isset($matches['_token_']) || $matches['_token_'] == '' )
                continue;

            $currentPos += strlen($matches['_token_']);
            $Token = $matches['_token_'];

            ## Regular Expression Token
            #
            #
            if ($Token === '~') {
                // ^ match any character that is not in list
                $pmatch = preg_match("/~(?P<_delimiter_>[^~]+)~/"
                    , $criteria
                    , $matches
                    , 0
                    , 0
                );
                if (! $pmatch )
                    throw new \RuntimeException('Miss usage of ~ ~ token.');

                $val['_regex_'] = $matches['_delimiter_'];
                $levelParts[$level][] = $val;

                $currentPos += strlen($matches[0])+2/*~x~*/;
            }
            ##
            # Variable
            # /:hash_id~\w{24}~
            #
            elseif ($Token === ':') {
                // ^$TOKENS  match any character that is not in list
                $pmatch = preg_match("(\G(?P<_name_>[^$TOKENS]+)(?:~(?P<_delimiter_>[^~]+)~)?:?)"
                    , $criteria
                    , $matches
                    , 0
                    , $currentPos
                );

                if (! $pmatch && !(isset($matches['_name_']) && isset($matches['_delimiter_'])) )
                    throw new \RuntimeException(
                        'Found empty parameter name or delimiter on (%s).'
                        , $criteria
                    );

                $parameter = $matches['_name_'];
                $val       = ['_parameter_' => $parameter];
                if (isset($matches['_delimiter_']))
                    $val[$parameter] = $matches['_delimiter_'];

                $levelParts[$level][] = $val;
                $currentPos += strlen($matches[0]);
            }

            ## Literal Token
            #  localhost\:port
            #
            elseif ($Token === '\\') {
                // Consider next character as Literal
                $nextChar = $currentPos += 1;
                $levelParts[$level][]   = ['_literal_' => $criteria[$nextChar]];
            }

            ## Optional Part
            #  /<u/:username~[a-zA-Z0-9._]+~><-:userid~\w{24}~>
            #
            elseif ($Token === '<') {
                if (! isset($va) ) {
                    $va = [];
                    $n = 0;
                }

                $n++;
                $va[$n] = [];
                $levelParts[$level][] = ['_optional_' => &$va[$n]];
                $level++;
                $levelParts[$level] = &$va[$n];
            }
            ## Close Optional ##
            elseif ($Token === '>') {
                unset($levelParts[$level]);
                $level--;

                if ($level < 0)
                    throw new \RuntimeException('Found closing bracket without matching opening bracket');
            }

            else
                // Recognized unused token return immanently
                $levelParts[$level][]   = ['_token_' => $Token];

        } // end while

        if ($level > 0)
            throw new \RuntimeException('Found unbalanced brackets');


        return ParsedCriteria::of($parts);
    }
225981
public function getNext()
    {
        for (; $this->idx < $this->count; ++$this->idx) {
            if (($this->tokens[$this->idx]->type !== Token::TYPE_WHITESPACE)
                && ($this->tokens[$this->idx]->type !== Token::TYPE_COMMENT)
            ) {
                return $this->tokens[$this->idx++];
            }
        }

        return null;
    }