Read the next token from the stream, comments and whitespace will be ignored.

@param SourceBuffer $generator
@return mixed A token as array or string or false on EOF.
431728
public function next(SourceBuffer $generator)
	{
		while(isset($this->tokens[$this->index]))
		{
			$tok = $this->tokens[$this->index++];
				
			if(is_array($tok))
			{
				switch($tok[0])
				{
					case T_COMMENT:
						$generator->append($tok);
						$this->comment = '';
						continue 2;
					case T_WHITESPACE:
						$generator->append($tok);
						continue 2;
					case T_DOC_COMMENT:
						$generator->append($tok);
						$this->comment = trim($tok[1]);
						continue 2;
					case T_CURLY_OPEN:
					case T_DOLLAR_OPEN_CURLY_BRACES:
					case T_STRING_VARNAME:
						$this->depth++;
						$this->comment = '';
						break;
					case T_CLOSE_TAG:
						$this->comment = '';
						break;
				}
			}
			else
			{
				switch($tok)
				{
					case '{':
						$this->depth++;
						$this->comment = '';
						break;
					case '}':
						$this->depth--;
						$this->comment = '';
						break;
					case ';':
					case ',':
						$this->comment = '';
						break;
				}
			}
				
			return $tok;
		}
	
		return false;
	}
129968
public static function parseSource($source, $namespace = '')
    {
        $matches = [];
        preg_match_all('#use[^;()]+?;#im', $source, $matches);
        $source = '<?php ' . implode('', $matches[0]);

        $builder = new AliasesBuilder;

        $states = [
            self::STATE_WAITING_FOR_USE => function ($token) use ($builder) {
                if (is_array($token) and $token[0] === T_USE) {
                    $builder->resetCurrent();
                    return AliasesParser::STATE_GATHERING;
                }
                return AliasesParser::STATE_WAITING_FOR_USE;
            },
            self::STATE_GATHERING => function ($token) use ($builder) {
                if (is_array($token)) {
                    if ($token[0] === T_STRING) {
                        $builder->appendToCurrent($token[1]);
                    } elseif ($token[0] === T_AS) {
                        return AliasesParser::STATE_IN_AS_PART;
                    }
                } else {
                    if ($token === ';') {
                        $builder->finishCurrent();
                        return AliasesParser::STATE_WAITING_FOR_USE;
                    } elseif ($token === ',') {
                        $builder->finishCurrent();
                    }
                }
                return AliasesParser::STATE_GATHERING;
            },
            self::STATE_IN_AS_PART => function ($token) use ($builder) {
                if (is_array($token)) {
                    if ($token[0] === T_STRING) {
                        $builder->setLast($token[1]);
                        $builder->finishCurrent();
                        return AliasesParser::STATE_JUST_FINISHED;
                    }
                }
                return AliasesParser::STATE_IN_AS_PART;
            },
            self::STATE_JUST_FINISHED => function ($token) use ($builder) {
                if ($token === ';') {
                    return AliasesParser::STATE_WAITING_FOR_USE;
                }
                return AliasesParser::STATE_GATHERING;
            },
        ];

        $state = $states[self::STATE_WAITING_FOR_USE];
        foreach (token_get_all($source) as $token) {
            $state = $states[$state($token)];
            if (is_array($token)) {
                $token[3] = token_name($token[0]);
            }
        }

        return $builder->getAliases($namespace);
    }
769977
public function readLine($token = "\n")
  {
    $this->connect();

    $line = Str::factory("");

    while(!$line->endsWith($token))
    {
      $this->waitForReadyRead();

      $data = @fgets($this->stream, 4096);

      Signal::getInstance()->emit(strtolower($this->getAdapterType()) . "DataRead", $data);

      if($data === FALSE)
      {
        if($line->count())
        {
          $line->append($token);
        }
        else
        {
          throw new Exception("connection to server '" . $this->config["host"] . ":" . $this->config["port"] . "' lost");
        }
      }
      else
      {
        $line->append($data);
      }
    }

    return $line->trim();
  }
995022
public function parse(Token $token)
    {
        $lineno = $token->getLine();
        $stream = $this->parser->getStream();

        $count = null;
        $vars = new ArrayExpression([], $lineno);
        $domain = null;
        $locale = null;
        if (!$stream->test(Token::BLOCK_END_TYPE)) {
            if ($stream->test('count')) {
                // {% trans count 5 %}
                $stream->next();
                $count = $this->parser->getExpressionParser()->parseExpression();
            }

            if ($stream->test('with')) {
                // {% trans with vars %}
                $stream->next();
                $vars = $this->parser->getExpressionParser()->parseExpression();
            }

            if ($stream->test('from')) {
                // {% trans from "messages" %}
                $stream->next();
                $domain = $this->parser->getExpressionParser()->parseExpression();
            }

            if ($stream->test('into')) {
                // {% trans into "fr" %}
                $stream->next();
                $locale = $this->parser->getExpressionParser()->parseExpression();
            } elseif (!$stream->test(Token::BLOCK_END_TYPE)) {
                throw new SyntaxError('Unexpected token. Twig was looking for the "with", "from", or "into" keyword.', $stream->getCurrent()->getLine(), $stream->getSourceContext());
            }
        }

        // {% trans %}message{% endtrans %}
        $stream->expect(Token::BLOCK_END_TYPE);
        $body = $this->parser->subparse([$this, 'decideTransFork'], true);

        if (!$body instanceof TextNode && !$body instanceof AbstractExpression) {
            throw new SyntaxError('A message inside a trans tag must be a simple text.', $body->getTemplateLine(), $stream->getSourceContext());
        }

        $stream->expect(Token::BLOCK_END_TYPE);

        return new TransNode($body, $domain, $count, $vars, $locale, $lineno, $this->getTag());
    }
966852
public function getNext()
    {
        if ($this->peeking) {
            $this->peeking = false;
            $this->used[] = $this->peeked;

            return $this->peeked;
        }

        if (!isset($this->tokens[$this->cursor])) {
            throw new InternalErrorException('Unexpected token stream end.');
        }

        return $this->tokens[$this->cursor++];
    }
473487
public function run(Generator $generator): string
    {
        return str_replace(
            $this->call,
            $generator->format($this->formatter, $this->parseArguments()),
            $this->definition
        );
    }
852845
protected function parseEncounteredToken(\Twig_Token $token, \Twig_TokenStream $stream)
    {
        switch ($token->getValue()) {
            case 'lang':
                $this->attributes['lang'] = $this->parseLangOption($token, $stream);
                break;

            case 'format':
                $this->attributes['format'] = $this->parseFormatOption($token, $stream);
                break;

            case 'start':
                $this->attributes['start'] = $this->parseStartOption($token, $stream);
                break;

            case 'mark':
                $this->attributes['mark'] = $this->parseMarkOption($token, $stream);
                break;

            case 'linenos':
                $this->attributes['linenos'] = $this->parseLinenosOption($token, $stream);
                break;

            case 'class':
                $this->attributes['class'] = $this->parseClassOption($token, $stream);
                break;

            case 'title':
                $this->attributes['title'] = $this->parseTitleOption($token, $stream);
                break;

            case 'link':
                $this->attributes['linkUrl'] = $this->parseLinkOption($token, $stream);
                break;

            case 'link_text':
                $this->attributes['linkText'] = $this->parseLinkTextOption($token, $stream);
                break;
        }
    }
294990
def get_token_stream(source: str) -> CommonTokenStream:
    
    lexer = LuaLexer(InputStream(source))
    stream = CommonTokenStream(lexer)
    return stream
792852
private function getValue(\Iterator $tokenIterator)
    {
        $message = '';
        $docToken = '';
        $docPart = '';

        for (; $tokenIterator->valid(); $tokenIterator->next()) {
            $t = $tokenIterator->current();
            if ('.' === $t) {
                // Concatenate with next token
                continue;
            }
            if (!isset($t[1])) {
                break;
            }

            switch ($t[0]) {
                case T_START_HEREDOC:
                    $docToken = $t[1];
                    break;
                case T_ENCAPSED_AND_WHITESPACE:
                case T_CONSTANT_ENCAPSED_STRING:
                    if ('' === $docToken) {
                        $message .= PhpStringTokenParser::parse($t[1]);
                    } else {
                        $docPart = $t[1];
                    }
                    break;
                case T_END_HEREDOC:
                    $message .= PhpStringTokenParser::parseDocString($docToken, $docPart);
                    $docToken = '';
                    $docPart = '';
                    break;
                case T_WHITESPACE:
                    break;
                default:
                    break 2;
            }
        }

        return $message;
    }
376598
public static function parse(Tokenizer $tokenizer, StreamReader $streamReader)
    {
        $v = new self;
        if (\strspn($streamReader->getByte(), "\x00\x09\x0A\x0C\x0D\x20()<>[]{}/%") === 0) {
            $v->value = (string) $tokenizer->getNextToken();
            return $v;
        }

        $v->value = '';
        return $v;
    }
370533
public static function readStream($source)
    {
        if (!$source) {
            return false;
        }
        try {
            $file = @fopen($source, 'rb');
            $content = '';
            do {
                $data = fread($file, 8192);
                if (strlen($data) == 0) {
                    break;
                }
                $content .= $data;
            } while (true);
            @fclose($file);
            return $content;
        } catch (\Exception $e) {
            return false;
        }
    }
758126
public function parse(Token $token)
    {
        $lineno = $token->getLine();
        $stream = $this->parser->getStream();

        $form = $this->parser->getExpressionParser()->parseExpression();
        $only = false;

        if ($this->parser->getStream()->test(Token::NAME_TYPE, 'with')) {
            $this->parser->getStream()->next();
            $resources = $this->parser->getExpressionParser()->parseExpression();

            if ($this->parser->getStream()->nextIf(Token::NAME_TYPE, 'only')) {
                $only = true;
            }
        } else {
            $resources = new ArrayExpression([], $stream->getCurrent()->getLine());
            do {
                $resources->addElement($this->parser->getExpressionParser()->parseExpression());
            } while (!$stream->test(Token::BLOCK_END_TYPE));
        }

        $stream->expect(Token::BLOCK_END_TYPE);

        return new FormThemeNode($form, $resources, $lineno, $this->getTag(), $only);
    }
264761
public function getIterator(): \Generator
    {
        while (!$this->stream->eof()) {
            $item = $this->stream->read($this->chunkSize);
            // @codeCoverageIgnoreStart
            //I don't know how to fabricate this manually, so I can't test it.
            if ($item === '') {
                continue;
            }
            // @codeCoverageIgnoreEnd
            yield $item;
        }
    }
774194
public static function parse(StreamReader $streamReader)
    {
        $bufferOffset = $streamReader->getOffset();

        /**
         * @var string $buffer
         * @var int $pos
         */
        while (true) {
            $buffer = $streamReader->getBuffer(false);
            $pos = \strpos($buffer, '>', $bufferOffset);
            if ($pos === false) {
                if (!$streamReader->increaseLength()) {
                    return false;
                }
                continue;
            }

            break;
        }

        $result = \substr($buffer, $bufferOffset, $pos - $bufferOffset);
        $streamReader->setOffset($pos + 1);

        $v = new self;
        $v->value = $result;

        return $v;
    }
1136220
public function getReadBufferAsync(&$length): \Generator
    {
        $buffer = yield $this->stream->getReadBuffer($l);
        $headers = '';
        $was_crlf = false;
        while (true) {
            $piece = yield $buffer->bufferRead(2);
            $headers .= $piece;
            if ($piece === "\n\r") { // Assume end of headers with \r\n\r\n
                $headers .= yield $buffer->bufferRead(1);
                break;
            }
            if ($was_crlf && $piece === "\r\n") {
                break;
            }
            $was_crlf = $piece === "\r\n";
        }
        $headers = explode("\r\n", $headers);

        list($protocol, $code, $description) = explode(' ', $headers[0], 3);
        list($protocol, $protocol_version) = explode('/', $protocol);
        if ($protocol !== 'HTTP') {
            throw new \danog\MadelineProto\Exception('Wrong protocol');
        }
        $code = (int) $code;
        unset($headers[0]);
        if (array_pop($headers).array_pop($headers) !== '') {
            throw new \danog\MadelineProto\Exception('Wrong last header');
        }
        foreach ($headers as $key => $current_header) {
            unset($headers[$key]);
            $current_header = explode(':', $current_header, 2);
            $headers[strtolower($current_header[0])] = trim($current_header[1]);
        }

        $close = $protocol === 'HTTP/1.0';
        if (isset($headers['connection'])) {
            $close = strtolower($headers['connection']) === 'close';
        }

        if ($code !== 200) {
            $read = '';
            if (isset($headers['content-length'])) {
                $read = yield $buffer->bufferRead((int) $headers['content-length']);
            }

            if ($close) {
                $this->disconnect();
                yield $this->connect($this->ctx);
            }

            \danog\MadelineProto\Logger::log($read);

            $this->code = $this->pack_signed_int(-$code);
            $length = 4;

            return $this;
        }

        if ($close) {
            $this->stream->disconnect();
            yield $this->stream->connect($this->ctx);
        }
        if (isset($headers['content-length'])) {
            $length = (int) $headers['content-length'];
        }

        return $buffer;
    }
214089
private function _tokens($source, $offset = 0, $additional = [], $embedded = false)
    {
        $result = new UnprocessedTokens();

        /** @var Language $language */
        foreach ($this->_rules($embedded) as $rule) {
            foreach ($rule->match($source) as $token) {
                $result->add($token, $offset);
            }
        }

        return $result->batch($additional);
    }
1197908
private void readTillEOL(StreamTokenizer tokenizer) throws IOException {
        while (tokenizer.nextToken() != StreamTokenizer.TT_EOL) {
        }

        tokenizer.pushBack();
    }