Force return of literal data as stream, if next token.

@see next()
965558
public function nextStream()
    {
        $changed = $this->_literalStream;
        $this->_literalStream = true;

        $out = $this->next();

        if ($changed) {
            $this->_literalStream = false;
        }

        return $out;
    }
1186641
@Nonnull
    public Token expect(@Nonnull String expected) throws IOException {
        if (!hasNext()) {
            throw eof("Expected %s: Got end of file", expected);
        }
        lastToken = unreadToken;
        unreadToken = null;
        return lastToken;
    }
162685
public function next()
    {
        $this->pointerAtStart = false;
        $this->pointerAtEnd = next($this->stream) === false;
    }
354673
Token next() throws ScanException {
        text = null;
        switch (c) {
            case '\n':
                return EOL;
            case ' ':
            case '\t':
                return whiteSpace();
            case ',':
                nextChar();
                return COMMA;
            case '/':
                nextChar();
                return SLASH;
            case '<':
                nextChar();
                return LESS;
            case '>':
                nextChar();
                return MORE;
            case '=':
                nextChar();
                return EQUALS;
            case '!':
                return pipe();
            case '"':
                return quotedText('"');
            case '\'':
                return quotedText('\'');
            default:
                return name();
        }

    }
532090
public function moveNext()
    {
        $this->peek = 0;
        $this->token = $this->next;
        $this->next = isset($this->tokens[++$this->i]) ? $this->tokens[$this->i] : null;

        return null !== $this->next;
    }
768256
@Nullable
    public JsonToken next() throws IOException, JsonException {
        if (unreadToken != null) {
            JsonToken tmp = unreadToken;
            unreadToken = null;
            return tmp;
        }

        while (lastChar >= 0) {
            if (lastChar == 0) {
                if (!readNextChar()) {
                    break;
                }
            }
            if (lastChar == JsonToken.kNewLine ||
                lastChar == JsonToken.kCarriageReturn ||
                lastChar == JsonToken.kSpace ||
                lastChar == JsonToken.kTab) {
                lastChar = 0;
                continue;
            }

            if (lastChar == JsonToken.kDoubleQuote) {
                return nextString();
            } else if (lastChar == '.' || lastChar == '-' || (lastChar >= '0' && lastChar <= '9')) {
                return nextNumber();
            } else if (lastChar == '_' ||
                       (lastChar >= 'a' && lastChar <= 'z') ||
                       (lastChar >= 'A' && lastChar <= 'Z')) {
                return nextToken();
            } else if (lastChar < 0x20 || lastChar >= 0x7F) {
                // UTF-8 characters are only allowed inside JSON string literals.
                throw newParseException("Illegal character in JSON structure: '\\u%04x'", lastChar);
            } else {
                return nextSymbol();
            }
        }

        return null;
    }
301087
final protected function parseRow(): void
    {
        // only parse the next row if only one in buffer
        if (\count($this->bufferedRow) > 1) {
            return;
        }
        if (feof($this->file)) {
            $this->currentRow = null;

            return;
        }

        $this->currentRowEnded = false;
        // read file until row is ended
        while (!$this->currentRowEnded && !feof($this->file)) {
            $data = fread($this->file, 1024);
            xml_parse($this->parser, $data);
        }
    }
1177710
public function next()
    {
        if (count($this->data) > 0) {
            array_shift($this->data);
        }
        ++$this->position;

        if (count($this->data) == 0 && !is_null($this->resumptionToken)) {
            $this->fetchMore();
        }

        if (count($this->data) == 0) {
            return null;
        }
    }
27793
def parse_stream(stream):
    """"""
    code = []

    for (line, col, (token, value)) in Tokenizer(stream).tokenize():
        if token == Tokenizer.STRING:
            value = '"' + value + '"'
        code.append(value)

    return code
133629
public function parse(TokenStream $stream)
    {
        if ($stream->match(Token::OPERATOR, array('!', '-'))) {
            $value = $stream->current()->value;
            $stream->next();
            $ret = new Op\Unary($value, $this->parse($stream));
        } elseif ($stream->match(Token::KEYWORD, array('true', 'false'))) {
            $ret = new Node\Expr\Literal($stream->current()->value === 'true');
            $stream->next();
        } elseif ($stream->match(Token::KEYWORD, 'null')) {
            $ret = new Node\Expr\Literal(null);
            $stream->next();
        } elseif ($stream->match(Token::IDENTIFIER)) {
            $name = $stream->current()->value;
            $stream->next();

            $ret = new Node\Expr\Variable($name);
        } elseif ($stream->match(Token::STRING)) {
            $ret = new Node\Expr\Str($stream->current()->value);
            $stream->next();
        } elseif ($stream->match(Token::NUMBER)) {
            $value = $stream->current()->value;
            if (strpos('.', $value) !== false) {
                $ret = new Node\Expr\Number((float)$value);
            } else {
                $ret = new Node\Expr\Number((int)$value);
            }
            $stream->next();
        } elseif ($stream->match(Token::OPERATOR, '(')) {
            $stream->next();
            $ret = new Node\Expr\Parens($this->parse($stream));
            $stream->expect(Token::OPERATOR, ')');
        } elseif ($stream->match(Token::OPERATOR, '[')) {
            $stream->next();
            $ret = new Node\Expr\ListNode();
            if (!$stream->match(Token::OPERATOR, ']')) {
                $ret->append($this->parse($stream));
                while ($stream->match(',')) {
                    $stream->next();
                    $ret->append($this->parse($stream));
                }
            }
            $stream->expect(Token::OPERATOR, ']');
        } else {
            $this->err($stream);
            return null;
        }

        if ($stream->valid()) {
            while ($stream->valid() && $stream->match(Token::OPERATOR, array('(', '.', '['))) {
                $type = $stream->current();

                $stream->next();
                if ($type->value === '(') {
                    $ret = new Node\Expr\Call($ret);
                    if (!$stream->match(Token::OPERATOR, ')')) {
                        do {
                            $arg = $this->parse($stream);
                            $ret->append($arg);

                            if ($stream->match(',')) {
                                $stream->next();
                            } else {
                                break;
                            }
                        } while (true);
                    }
                    $stream->expect(Token::OPERATOR, ')');
                } else {
                    $ret = new Node\Expr\Subscript($ret);

                    if ($type->value === '.') {
                        $token = $stream->expect(Token::IDENTIFIER);
                        $ret->append(new Node\Expr\Str($token->value));
                    } else {
                        $ret->append($this->parse($stream));
                    }

                    switch ($type->value) {
                        case '[':
                            $stream->expect(Token::OPERATOR, ']');
                            break;
                    }
                }
            }
        }

        if ($stream->valid()) {
            if ($stream->match(Token::OPERATOR, self::$INFIX_BINARY) || $stream->match(Token::KEYWORD, 'in')) {
                $value = $stream->current()->value;
                $stream->next();
                $ret = new Op\Binary($value, $ret, $this->parse($stream));
            } elseif ($stream->match(Token::OPERATOR, '?')) {
                $stream->next();
                if ($stream->match(Token::OPERATOR, ':')) {
                    $then = null;
                } else {
                    $then = $this->parse($stream);
                }

                if ($stream->valid() && $stream->match(Token::OPERATOR, ':')) {
                    $stream->next();
                    $else = $this->parse($stream);
                } else {
                    $else = null;
                }
                $ret = new Op\Ternary('?', $ret, $then, $else);
            }
        }

        // little syntactic sugar for function calls without parentheses:
        $allowInlineCallTokens = array(Token::IDENTIFIER, Token::STRING, Token::NUMBER);
        if ($stream->valid() && ($stream->match($allowInlineCallTokens))) {
            $ret = new Node\Expr\Call($ret);
            $ret->append($this->parse($stream));
        }

        return $ret;
    }
1324840
function streamToString(callback) {
  var contents = '';

  return through(
    function (chunk, enc, next) {
      contents += chunk.toString('utf8');
      next();
    },
    function (next) { // flush function
      callback.bind(this)(contents, next);
    }
  );
}
1306177
private final Return returnStatement(Data data) throws TemplateException {
	if (!data.srcCode.forwardIfCurrentAndNoVarExt("return")) return null;

	Position line = data.srcCode.getPosition();
	Return rtn;

	comments(data);
	if (checkSemiColonLineFeed(data, false, false, false)) rtn = new Return(data.factory, line, data.srcCode.getPosition());
	else {
	    Expression expr = expression(data);
	    checkSemiColonLineFeed(data, true, true, false);
	    rtn = new Return(expr, line, data.srcCode.getPosition());
	}
	comments(data);

	return rtn;
    }
1199757
function tokenBase(stream, state) {
    var ch = stream.next(), 
        mightBeFunction = false,
        isEQName = isEQNameAhead(stream);
    
    // an XML tag (if not in some sub, chained tokenizer)
    if (ch == "<") {
      if(stream.match("!--", true))
        return chain(stream, state, tokenXMLComment);
        
      if(stream.match("![CDATA", false)) {
        state.tokenize = tokenCDATA;
        return ret("tag", "tag");
      }
      
      if(stream.match("?", false)) {
        return chain(stream, state, tokenPreProcessing);
      }
      
      var isclose = stream.eat("/");
      stream.eatSpace();
      var tagName = "", c;
      while ((c = stream.eat(/[^\s\u00a0=<>\"\'\/?]/))) tagName += c;
      
      return chain(stream, state, tokenTag(tagName, isclose));
    }
    // start code block
    else if(ch == "{") {
      pushStateStack(state,{ type: "codeblock"});
      return ret("", "");
    }
    // end code block
    else if(ch == "}") {
      popStateStack(state);
      return ret("", "");
    }
    // if we're in an XML block
    else if(isInXmlBlock(state)) {
      if(ch == ">")
        return ret("tag", "tag");
      else if(ch == "/" && stream.eat(">")) {
        popStateStack(state);
        return ret("tag", "tag");
      }
      else  
        return ret("word", "variable");
    }
    // if a number
    else if (/\d/.test(ch)) {
      stream.match(/^\d*(?:\.\d*)?(?:E[+\-]?\d+)?/);
      return ret("number", "atom");
    }
    // comment start
    else if (ch === "(" && stream.eat(":")) {
      pushStateStack(state, { type: "comment"});
      return chain(stream, state, tokenComment);
    }
    // quoted string
    else if (  !isEQName && (ch === '"' || ch === "'"))
      return chain(stream, state, tokenString(ch));
    // variable
    else if(ch === "$") {
      return chain(stream, state, tokenVariable);
    }
    // assignment
    else if(ch ===":" && stream.eat("=")) {
      return ret("operator", "keyword");
    }
    // open paren
    else if(ch === "(") {
      pushStateStack(state, { type: "paren"});
      return ret("", "");
    }
    // close paren
    else if(ch === ")") {
      popStateStack(state);
      return ret("", "");
    }
    // open paren
    else if(ch === "[") {
      pushStateStack(state, { type: "bracket"});
      return ret("", "");
    }
    // close paren
    else if(ch === "]") {
      popStateStack(state);
      return ret("", "");
    }
    else {
      var known = keywords.propertyIsEnumerable(ch) && keywords[ch];

      // if there's a EQName ahead, consume the rest of the string portion, it's likely a function
      if(isEQName && ch === '\"') while(stream.next() !== '"'){}
      if(isEQName && ch === '\'') while(stream.next() !== '\''){}
      
      // gobble up a word if the character is not known
      if(!known) stream.eatWhile(/[\w\$_-]/);
      
      // gobble a colon in the case that is a lib func type call fn:doc
      var foundColon = stream.eat(":");
      
      // if there's not a second colon, gobble another word. Otherwise, it's probably an axis specifier
      // which should get matched as a keyword
      if(!stream.eat(":") && foundColon) {
        stream.eatWhile(/[\w\$_-]/);
      }
      // if the next non whitespace character is an open paren, this is probably a function (if not a keyword of other sort)
      if(stream.match(/^[ \t]*\(/, false)) {
        mightBeFunction = true;
      }
      // is the word a keyword?
      var word = stream.current();
      known = keywords.propertyIsEnumerable(word) && keywords[word];
      
      // if we think it's a function call but not yet known, 
      // set style to variable for now for lack of something better
      if(mightBeFunction && !known) known = {type: "function_call", style: "variable def"};
      
      // if the previous word was element, attribute, axis specifier, this word should be the name of that
      if(isInXmlConstructor(state)) {
        popStateStack(state);
        return ret("word", "variable", word);
      }
      // as previously checked, if the word is element,attribute, axis specifier, call it an "xmlconstructor" and 
      // push the stack so we know to look for it on the next word
      if(word == "element" || word == "attribute" || known.type == "axis_specifier") pushStateStack(state, {type: "xmlconstructor"});
      
      // if the word is known, return the details of that else just call this a generic 'word'
      return known ? ret(known.type, known.style, word) :
                     ret("word", "variable", word);
    }
  }
76862
public function nextToken()
    {
        if ($this->_input === null) {
            return null;
        }

        do {
            if (! preg_match('/[a-zA-Z0-9]+/', $this->_input, $match, PREG_OFFSET_CAPTURE, $this->_position)) {
                // It covers both cases a) there are no matches (preg_match(...) === 0)
                // b) error occured (preg_match(...) === FALSE)
                return null;
            }

            $str = $match[0][0];
            $pos = $match[0][1];
            $endpos = $pos + strlen($str);

            $this->_position = $endpos;

            $token = $this->normalize(new Zend_Search_Lucene_Analysis_Token($str, $pos, $endpos));
        } while ($token === null); // try again if token is skipped

        return $token;
    }
1288386
def load(stream):
    """"""
    if isinstance(stream, str):
        string = stream
    else:
        string = stream.read()
    tokens = tokenize(string)
    return parse(tokens)
771013
public final void nextToken() {
        skipIgnoredToken();
        if (isVariableBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanVariable();
        } else if (isNCharBegin()) {
            currentToken = new Tokenizer(input, dictionary, ++offset).scanChars();
        } else if (isIdentifierBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanIdentifier();
        } else if (isHexDecimalBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanHexDecimal();
        } else if (isNumberBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanNumber();
        } else if (isSymbolBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanSymbol();
        } else if (isCharsBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanChars();
        } else if (isEnd()) {
            currentToken = new Token(Assist.END, "", offset);
        } else {
            throw new SQLParsingException(this, Assist.ERROR);
        }
        offset = currentToken.getEndPosition();
    }
1154965
public function parse(AbstractStream $stream)
    {
        if (!($this->dataSet instanceof DataSet)) {
            $this->dataSet = new DataSet;
        }

        $this->rewind();
        do {
            $field = $this->current();
            $field->setDataSet($this->getDataSet());
            if ($field instanceof Conditional) {
                $field = $field->resolveField();
            }
            if ($field instanceof self) {
                $this->dataSet->push($this->key());
                $field->parse($stream);
                $this->dataSet->pop();
            } else {
                $this->dataSet->setValue($this->key(), $field->parse($stream));
            }
            $this->next();
        } while ($this->valid());

        if (isset($this->assert)) {
            $this->validate($this->dataSet->getValueByCurrentPath());
        }

        return $this->dataSet->getData();
    }