Returns the next token in the stream.  Note that unlike other iterators,
this method is {@code O(n)} in complexity where {@code n} is the number
of unique words starting the set of compound tokens recognized by this
iterator.
427310
public String next() {
	if (!hasNext()) {
	    throw new NoSuchElementException();
	}

	String token = tokenizer.next();

	// Determine whether the next token to return could be the start of a
	// recognized compound token
	CompoundTokens compounds = compoundTokens.get(token);
	
	// Determine how many tokens are needed at most and see if any grouping
	// exists
	if (compounds != null) {
	    List<String> nextTokens = tokenizer.peek(compounds.maxTokens());
	    Duple<Integer,String> compound = compounds.findMatch(nextTokens);
	    if (compound != null) {
		// shift off the number of extra tokens in the compound
		for (int i = 0; i < compound.x; ++i) 
		    tokenizer.next();
		return compound.y;
	    }
	    else return token;
	}
	
	// If there was no possibility of grouping this token, then return it by
	// itself
	return token;
    }
1167337
public T getNext() throws IOException {
    try {
      return mapper.readValue(jsonParser, clazz);
    } catch (EOFException e) {
      return null;
    }
  }
970875
private Token nextToken() {
        skipSpaces();
        if (hasNextChar()) {
            char ch = nextChar();
            if (Character.isJavaIdentifierStart(ch)) {
                StringBuffer buf = new StringBuffer();
                buf.append(ch);
                while ((ch = peekChar()) != -1
                        && Character.isJavaIdentifierPart(ch)) {
                    buf.append(ch);
                    nextChar();
                }
                return new Id(buf.toString());
            }

            if (ch == '\'' || ch == '"') {
                return parseQuotedChars(ch);
            } else {
                // For now...
                return new Char(ch);
            }
        }
        return null;
    }
1163510
@Nullable
    public Token next() throws IOException {
        if (unreadToken != null) {
            lastToken = unreadToken;
            unreadToken = null;
            return lastToken;
        }

        while (lastChar >= 0) {
            if (lastChar == 0) {
                if (!readNextChar()) {
                    break;
                }
            }
            if (lastChar == Token.kNewLine ||
                lastChar == Token.kCarriageReturn ||
                lastChar == Token.kSpace ||
                lastChar == Token.kTab) {
                lastChar = 0;
                continue;
            }

            if (lastChar == Token.kShellComment) {
                getRestOfLine();
                continue;
            }

            if (lastChar == Token.kLiteralDoubleQuote || lastChar == Token.kLiteralQuote) {
                lastToken = nextStringLiteral((char) lastChar);
                return lastToken;
            } else if (lastChar == '.' || lastChar == '-' || (lastChar >= '0' && lastChar <= '9')) {
                lastToken = nextNumber();
                return lastToken;
            } else if ('_' == lastChar ||
                       ('a' <= lastChar && lastChar <= 'z') ||
                       ('A' <= lastChar && lastChar <= 'Z')) {
                lastToken = nextToken();
                return lastToken;
            } else if (lastChar < 0x20 || lastChar >= 0x7F) {
                // non-ASCII UTF-8 characters are only allowed inside JSON string literals.
                throw failure(lineNo, linePos, 1, "Unknown token initiator '%s'", Strings.escape((char) lastChar));
            } else {
                lastToken = nextSymbol();
                return lastToken;
            }
        }

        return null;
    }
1294505
@Override
	public <C extends Compound> Sequence<C> getSubSequence(Sequence<C> sequence) {
		if(isCircular()) {
			List<Sequence<C>> sequences = new ArrayList<Sequence<C>>();
			for(Location l: this) {
				sequences.add(l.getSubSequence(sequence));
			}
			return new JoiningSequenceReader<C>(sequence.getCompoundSet(), sequences);
		}
		return reverseSequence(sequence.getSubSequence(
				getStart().getPosition(), getEnd().getPosition()));
	}
1036076
public Token next() {
        if (!splitTokens.isEmpty()) {
            return splitTokens.remove();
        }
        var value = stringIterator.next();

        if (argumentEscapeEncountered) {
            return new ArgumentToken(value);
        }

        if (Objects.equals(value, argumentTerminator)) {
            argumentTerminator = null;
            return new ArgumentToken(value);
        }
        if (argumentTerminator != null) {
            return new ArgumentToken(value);
        }
        if (isArgumentEscape(value)) {
            argumentEscapeEncountered = true;
            return next();
        }
        if (isSwitch(value)) {
            if (isShortSwitchList(value)) {
                splitTokens.addAll(splitSwitchTokens(value));
                return splitTokens.remove();
            } else {
                return new SwitchToken(value.substring(2), value);
            }
        } else {
            return new ArgumentToken(value);
        }
    }
1290888
public com.davidbracewell.parsing.ParserToken next() throws java.io.IOException,     com.davidbracewell.parsing.ParseException
 {
    int zzInput;
    int zzAction;

    // cached fields:
    int zzCurrentPosL;
    int zzMarkedPosL;
    int zzEndReadL = zzEndRead;
    char [] zzBufferL = zzBuffer;
    char [] zzCMapL = ZZ_CMAP;

    int [] zzTransL = ZZ_TRANS;
    int [] zzRowMapL = ZZ_ROWMAP;
    int [] zzAttrL = ZZ_ATTRIBUTE;

    while (true) {
      zzMarkedPosL = zzMarkedPos;

      yychar+= zzMarkedPosL-zzStartRead;

      boolean zzR = false;
      int zzCh;
      int zzCharCount;
      for (zzCurrentPosL = zzStartRead  ;
           zzCurrentPosL < zzMarkedPosL ;
           zzCurrentPosL += zzCharCount ) {
        zzCh = Character.codePointAt(zzBufferL, zzCurrentPosL, zzMarkedPosL);
        zzCharCount = Character.charCount(zzCh);
        switch (zzCh) {
        case '\u000B':
        case '\u000C':
        case '\u0085':
        case '\u2028':
        case '\u2029':
          yyline++;
          zzR = false;
          break;
        case '\r':
          yyline++;
          zzR = true;
          break;
        case '\n':
          if (zzR)
            zzR = false;
          else {
            yyline++;
          }
          break;
        default:
          zzR = false;
        }
      }

      if (zzR) {
        // peek one character ahead if it is \n (if we have counted one line too much)
        boolean zzPeek;
        if (zzMarkedPosL < zzEndReadL)
          zzPeek = zzBufferL[zzMarkedPosL] == '\n';
        else if (zzAtEOF)
          zzPeek = false;
        else {
          boolean eof = zzRefill();
          zzEndReadL = zzEndRead;
          zzMarkedPosL = zzMarkedPos;
          zzBufferL = zzBuffer;
          if (eof) 
            zzPeek = false;
          else 
            zzPeek = zzBufferL[zzMarkedPosL] == '\n';
        }
        if (zzPeek) yyline--;
      }
      zzAction = -1;

      zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
  
      zzState = ZZ_LEXSTATE[zzLexicalState];

      // set up zzAction for empty match case:
      int zzAttributes = zzAttrL[zzState];
      if ( (zzAttributes & 1) == 1 ) {
        zzAction = zzState;
      }


      zzForAction: {
        while (true) {
    
          if (zzCurrentPosL < zzEndReadL) {
            zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL);
            zzCurrentPosL += Character.charCount(zzInput);
          }
          else if (zzAtEOF) {
            zzInput = YYEOF;
            break zzForAction;
          }
          else {
            // store back cached positions
            zzCurrentPos  = zzCurrentPosL;
            zzMarkedPos   = zzMarkedPosL;
            boolean eof = zzRefill();
            // get translated positions and possibly new buffer
            zzCurrentPosL  = zzCurrentPos;
            zzMarkedPosL   = zzMarkedPos;
            zzBufferL      = zzBuffer;
            zzEndReadL     = zzEndRead;
            if (eof) {
              zzInput = YYEOF;
              break zzForAction;
            }
            else {
              zzInput = Character.codePointAt(zzBufferL, zzCurrentPosL, zzEndReadL);
              zzCurrentPosL += Character.charCount(zzInput);
            }
          }
          int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
          if (zzNext == -1) break zzForAction;
          zzState = zzNext;

          zzAttributes = zzAttrL[zzState];
          if ( (zzAttributes & 1) == 1 ) {
            zzAction = zzState;
            zzMarkedPosL = zzCurrentPosL;
            if ( (zzAttributes & 8) == 8 ) break zzForAction;
          }

        }
      }

      // store back cached position
      zzMarkedPos = zzMarkedPosL;

      if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
        zzAtEOF = true;
        return null;
      }
      else {
        switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
          case 1: 
            { yybegin(stack.pop());  return new ParserToken(yytext(),ConfigTokenType.VALUE);
            }
          case 19: break;
          case 2: 
            { throw new com.davidbracewell.parsing.ParseException("UNEXPECTED TOKEN \"" + yytext() +"\" at line: " + yyline + " char offset: " + yychar + " state: " + yystate());
            }
          case 20: break;
          case 3: 
            { 
            }
          case 21: break;
          case 4: 
            { stack.push(YYINITIAL);yybegin(SECTION_PROPERTY);
            }
          case 22: break;
          case 5: 
            { yybegin(stack.pop()); return new ParserToken(yytext(),ConfigTokenType.END_SECTION);
            }
          case 23: break;
          case 6: 
            { yybegin(SECTION_PROPERTY); return new ParserToken(yytext(),ConfigTokenType.VALUE);
            }
          case 24: break;
          case 7: 
            { yybegin(YYINITIAL); return new ParserToken(yytext(),ConfigTokenType.VALUE);
            }
          case 25: break;
          case 8: 
            { yybegin(SECTION_PROPERTY);
            }
          case 26: break;
          case 9: 
            { yybegin(VALUE);
            }
          case 27: break;
          case 10: 
            { yybegin(SECTION_VALUE);
            }
          case 28: break;
          case 11: 
            // general lookahead, find correct zzMarkedPos
            { int zzFState = 9;
              int zzFPos = zzStartRead;
              if (zzFin.length <= zzBufferL.length) { zzFin = new boolean[zzBufferL.length+1]; }
              boolean zzFinL[] = zzFin;
              while (zzFState != -1 && zzFPos < zzMarkedPos) {
                zzFinL[zzFPos] = ((zzAttrL[zzFState] & 1) == 1);
                zzInput = Character.codePointAt(zzBufferL, zzFPos, zzMarkedPos);
                zzFPos += Character.charCount(zzInput);
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              }
              if (zzFState != -1) { zzFinL[zzFPos++] = ((zzAttrL[zzFState] & 1) == 1); } 
              while (zzFPos <= zzMarkedPos) {
                zzFinL[zzFPos++] = false;
              }

              zzFState = 12;
              zzFPos = zzMarkedPos;
              while (!zzFinL[zzFPos] || (zzAttrL[zzFState] & 1) != 1) {
                zzInput = Character.codePointBefore(zzBufferL, zzFPos, zzStartRead);
                zzFPos -= Character.charCount(zzInput);
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              };
              zzMarkedPos = zzFPos;
            }
            { stack.push(YYINITIAL);yybegin(START_VALUE); return new ParserToken(yytext(),ConfigTokenType.PROPERTY);
            }
          case 29: break;
          case 12: 
            // general lookahead, find correct zzMarkedPos
            { int zzFState = 9;
              int zzFPos = zzStartRead;
              if (zzFin.length <= zzBufferL.length) { zzFin = new boolean[zzBufferL.length+1]; }
              boolean zzFinL[] = zzFin;
              while (zzFState != -1 && zzFPos < zzMarkedPos) {
                zzFinL[zzFPos] = ((zzAttrL[zzFState] & 1) == 1);
                zzInput = Character.codePointAt(zzBufferL, zzFPos, zzMarkedPos);
                zzFPos += Character.charCount(zzInput);
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              }
              if (zzFState != -1) { zzFinL[zzFPos++] = ((zzAttrL[zzFState] & 1) == 1); } 
              while (zzFPos <= zzMarkedPos) {
                zzFinL[zzFPos++] = false;
              }

              zzFState = 10;
              zzFPos = zzMarkedPos;
              while (!zzFinL[zzFPos] || (zzAttrL[zzFState] & 1) != 1) {
                zzInput = Character.codePointBefore(zzBufferL, zzFPos, zzStartRead);
                zzFPos -= Character.charCount(zzInput);
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              };
              zzMarkedPos = zzFPos;
            }
            { stack.push(YYINITIAL);yybegin(START_SECTION); return new ParserToken(yytext(),ConfigTokenType.SECTION_HEADER);
            }
          case 30: break;
          case 13: 
            // general lookahead, find correct zzMarkedPos
            { int zzFState = 9;
              int zzFPos = zzStartRead;
              if (zzFin.length <= zzBufferL.length) { zzFin = new boolean[zzBufferL.length+1]; }
              boolean zzFinL[] = zzFin;
              while (zzFState != -1 && zzFPos < zzMarkedPos) {
                zzFinL[zzFPos] = ((zzAttrL[zzFState] & 1) == 1);
                zzInput = Character.codePointAt(zzBufferL, zzFPos, zzMarkedPos);
                zzFPos += Character.charCount(zzInput);
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              }
              if (zzFState != -1) { zzFinL[zzFPos++] = ((zzAttrL[zzFState] & 1) == 1); } 
              while (zzFPos <= zzMarkedPos) {
                zzFinL[zzFPos++] = false;
              }

              zzFState = 12;
              zzFPos = zzMarkedPos;
              while (!zzFinL[zzFPos] || (zzAttrL[zzFState] & 1) != 1) {
                zzInput = Character.codePointBefore(zzBufferL, zzFPos, zzStartRead);
                zzFPos -= Character.charCount(zzInput);
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              };
              zzMarkedPos = zzFPos;
            }
            { yybegin(START_SECTION_VALUE); return new ParserToken(yytext(),ConfigTokenType.PROPERTY);
            }
          case 31: break;
          case 14: 
            // general lookahead, find correct zzMarkedPos
            { int zzFState = 9;
              int zzFPos = zzStartRead;
              if (zzFin.length <= zzBufferL.length) { zzFin = new boolean[zzBufferL.length+1]; }
              boolean zzFinL[] = zzFin;
              while (zzFState != -1 && zzFPos < zzMarkedPos) {
                zzFinL[zzFPos] = ((zzAttrL[zzFState] & 1) == 1);
                zzInput = Character.codePointAt(zzBufferL, zzFPos, zzMarkedPos);
                zzFPos += Character.charCount(zzInput);
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              }
              if (zzFState != -1) { zzFinL[zzFPos++] = ((zzAttrL[zzFState] & 1) == 1); } 
              while (zzFPos <= zzMarkedPos) {
                zzFinL[zzFPos++] = false;
              }

              zzFState = 10;
              zzFPos = zzMarkedPos;
              while (!zzFinL[zzFPos] || (zzAttrL[zzFState] & 1) != 1) {
                zzInput = Character.codePointBefore(zzBufferL, zzFPos, zzStartRead);
                zzFPos -= Character.charCount(zzInput);
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              };
              zzMarkedPos = zzFPos;
            }
            { stack.push(SECTION_PROPERTY);yybegin(START_SECTION); return new ParserToken(yytext(),ConfigTokenType.SECTION_HEADER);
            }
          case 32: break;
          case 15: 
            // general lookahead, find correct zzMarkedPos
            { int zzFState = 9;
              int zzFPos = zzStartRead;
              if (zzFin.length <= zzBufferL.length) { zzFin = new boolean[zzBufferL.length+1]; }
              boolean zzFinL[] = zzFin;
              while (zzFState != -1 && zzFPos < zzMarkedPos) {
                zzFinL[zzFPos] = ((zzAttrL[zzFState] & 1) == 1);
                zzInput = Character.codePointAt(zzBufferL, zzFPos, zzMarkedPos);
                zzFPos += Character.charCount(zzInput);
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              }
              if (zzFState != -1) { zzFinL[zzFPos++] = ((zzAttrL[zzFState] & 1) == 1); } 
              while (zzFPos <= zzMarkedPos) {
                zzFinL[zzFPos++] = false;
              }

              zzFState = 11;
              zzFPos = zzMarkedPos;
              while (!zzFinL[zzFPos] || (zzAttrL[zzFState] & 1) != 1) {
                zzInput = Character.codePointBefore(zzBufferL, zzFPos, zzStartRead);
                zzFPos -= Character.charCount(zzInput);
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              };
              zzMarkedPos = zzFPos;
            }
            { stack.push(YYINITIAL);yybegin(APPENDER); return new ParserToken(yytext(),ConfigTokenType.APPEND_PROPERTY);
            }
          case 33: break;
          case 16: 
            // general lookahead, find correct zzMarkedPos
            { int zzFState = 9;
              int zzFPos = zzStartRead;
              if (zzFin.length <= zzBufferL.length) { zzFin = new boolean[zzBufferL.length+1]; }
              boolean zzFinL[] = zzFin;
              while (zzFState != -1 && zzFPos < zzMarkedPos) {
                zzFinL[zzFPos] = ((zzAttrL[zzFState] & 1) == 1);
                zzInput = Character.codePointAt(zzBufferL, zzFPos, zzMarkedPos);
                zzFPos += Character.charCount(zzInput);
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              }
              if (zzFState != -1) { zzFinL[zzFPos++] = ((zzAttrL[zzFState] & 1) == 1); } 
              while (zzFPos <= zzMarkedPos) {
                zzFinL[zzFPos++] = false;
              }

              zzFState = 11;
              zzFPos = zzMarkedPos;
              while (!zzFinL[zzFPos] || (zzAttrL[zzFState] & 1) != 1) {
                zzInput = Character.codePointBefore(zzBufferL, zzFPos, zzStartRead);
                zzFPos -= Character.charCount(zzInput);
                zzFState = zzTransL[ zzRowMapL[zzFState] + zzCMapL[zzInput] ];
              };
              zzMarkedPos = zzFPos;
            }
            { stack.push(SECTION_PROPERTY); yybegin(APPENDER); return new ParserToken(yytext(),ConfigTokenType.APPEND_PROPERTY);
            }
          case 34: break;
          case 17: 
            { yybegin(IMPORT); return new ParserToken(yytext(),ConfigTokenType.IMPORT);
            }
          case 35: break;
          case 18: 
            { yybegin(SCRIPT); return new ParserToken(yytext(),ConfigTokenType.SCRIPT);
            }
          case 36: break;
          default:
            zzScanError(ZZ_NO_MATCH);
        }
      }
    }
  }
591642
@Override
    public Token next() {
        if (!hasNext())
            throw new NoSuchElementException();
        Token t = this.tok;
        this.tok = null;
        return t;
    }
1198592
public Token lookahead() {
        Token current = token;
        if (current.next == null) {
            current.next = token_source.getNextToken();
        }
        return current.next;
    }
1366452
public Type readNextToken() throws IOException {
		int c;
		if (currentCharacter >= 0 && !Character.isWhitespace(currentCharacter)) {
			//there's still a character left from the last step
			c = currentCharacter;
			currentCharacter = -1;
		} else {
			//skip whitespace characters
			c = skipWhitespace();
		}
		if (c < 0) {
			return null;
		}
		
		//handle character
		if (c =='{') {
			currentTokenType = Type.START_OBJECT;
		} else if (c == '}') {
			currentTokenType = Type.END_OBJECT;
		} else if (c == '[') {
			currentTokenType = Type.START_ARRAY;
		} else if (c == ']') {
			currentTokenType = Type.END_ARRAY;
		} else if (c == ':') {
			currentTokenType = Type.COLON;
		} else if (c == ',') {
			currentTokenType = Type.COMMA;
		} else if (c == '"') {
			currentTokenType = Type.STRING;
		} else if (c == '-' || (c >= '0' && c<= '9')) {
			currentTokenType = Type.NUMBER;
			//the next token is a number. save the last character read because
			//readNumber() will need it.
			currentCharacter = c;
		} else if (c == 't') {
			int c2 = r.read();
			int c3 = r.read();
			int c4 = r.read();
			if (c2 == 'r' && c3 == 'u' & c4 == 'e') {
				currentTokenType = Type.TRUE;
			} else {
				currentTokenType = null;
			}
		} else if (c == 'f') {
			int c2 = r.read();
			int c3 = r.read();
			int c4 = r.read();
			int c5 = r.read();
			if (c2 == 'a' && c3 == 'l' & c4 == 's' && c5 == 'e') {
				currentTokenType = Type.FALSE;
			} else {
				currentTokenType = null;
			}
		} else if (c == 'n') {
			int c2 = r.read();
			int c3 = r.read();
			int c4 = r.read();
			if (c2 == 'u' && c3 == 'l' & c4 == 'l') {
				currentTokenType = Type.NULL;
			} else {
				currentTokenType = null;
			}
		} else {
			currentTokenType = null;
		}
		
		if (currentTokenType == null) {
			throw new IllegalStateException("Unrecognized token: " + (char)c);
		}
		
		return currentTokenType;
	}
224718
private boolean streamHasMoreTokens() {
        if (streamTokenizer.ttype != StreamTokenizer.TT_EOF) {
            try {
                streamTokenizer.nextToken();
            } catch (IOException e1) {
                throw new RuntimeException(e1);
            }
        }
        return streamTokenizer.ttype != StreamTokenizer.TT_EOF && streamTokenizer.ttype != -1;
    }
680148
@Override
    public List<String> getTokens() {
        //List<String> tokens = new ArrayList<>();
        if (!tokens.isEmpty())
            return tokens;

        log.info("Starting prebuffering...");
        while (streamHasMoreTokens()) {
            tokens.add(nextTokenFromStream());
        }
        log.info("Tokens prefetch finished. Tokens size: [" + tokens.size() + "]");
        return tokens;
    }
1329124
@Requires("nextToken == null")
  @Ensures("result == (nextToken != null)")
  protected boolean lex() throws IOException {
    int startOffset = currentOffset;
    StringBuilder buffer = new StringBuilder();
    int c = readChar(true);
    if (c == -1) {
      return false;
    }
    buffer.append((char) c);
    switch (c) {
      case '/':
        /* Comments. */
        c = readChar(true);
        switch (c) {
          case '/':
            buffer.append((char) c);
            do {
              c = readChar(false);
              buffer.append((char) c);
            } while (c != '\n');
            nextToken = new Token(TokenKind.COMMENT, buffer.toString(),
                                  startOffset);
            break;
          case '*':
            buffer.append((char) c);
            for (;;) {
              c = readChar(false);
              buffer.append((char) c);
              if (c == '*') {
                c = readChar(false);
                buffer.append((char) c);
                if (c == '/') {
                  break;
                }
              }
            }
            nextToken = new Token(TokenKind.COMMENT, buffer.toString(),
                                  startOffset);
            break;
          default:
            unreadChar(c);
            nextToken = new Token(TokenKind.SYMBOL, buffer.toString(),
                                  startOffset);
        }
        break;

      case '\'':
      case '"':
        /* Quoted strings. */
        int delim = c;
        for (;;) {
          c = readChar(false);
          buffer.append((char) c);
          if (c == delim) {
            break;
          }
          if (c == '\\') {
            buffer.append((char) readChar(false));
          }
        }
        nextToken = new Token(TokenKind.QUOTE, buffer.toString(), startOffset);
        break;

      default:
        if (Character.isJavaIdentifierStart(c)) {
          /* Identifiers. */
          while ((c = readChar(true)) != -1
                 && Character.isJavaIdentifierPart(c)) {
            buffer.append((char) c);
          }
          unreadChar(c);
          nextToken = new Token(TokenKind.WORD, buffer.toString(), startOffset);
        } else if (Character.isWhitespace(c)) {
          /* White space. */
          while ((c = readChar(true)) != -1 && Character.isWhitespace(c)) {
            buffer.append((char) c);
          }
          unreadChar(c);
          nextToken = new Token(TokenKind.SPACE, buffer.toString(),
                                startOffset);
        } else {
          /* Symbol. */
          nextToken = new Token(TokenKind.SYMBOL, buffer.toString(),
                                startOffset);
        }
    }
    return true;
  }
1084467
private final void nextToken()
  {

    if (m_queueMark < m_ops.getTokenQueueSize())
    {
      m_token = (String) m_ops.m_tokenQueue.elementAt(m_queueMark++);
      m_tokenChar = m_token.charAt(0);
    }
    else
    {
      m_token = null;
      m_tokenChar = 0;
    }
  }
768256
@Nullable
    public JsonToken next() throws IOException, JsonException {
        if (unreadToken != null) {
            JsonToken tmp = unreadToken;
            unreadToken = null;
            return tmp;
        }

        while (lastChar >= 0) {
            if (lastChar == 0) {
                if (!readNextChar()) {
                    break;
                }
            }
            if (lastChar == JsonToken.kNewLine ||
                lastChar == JsonToken.kCarriageReturn ||
                lastChar == JsonToken.kSpace ||
                lastChar == JsonToken.kTab) {
                lastChar = 0;
                continue;
            }

            if (lastChar == JsonToken.kDoubleQuote) {
                return nextString();
            } else if (lastChar == '.' || lastChar == '-' || (lastChar >= '0' && lastChar <= '9')) {
                return nextNumber();
            } else if (lastChar == '_' ||
                       (lastChar >= 'a' && lastChar <= 'z') ||
                       (lastChar >= 'A' && lastChar <= 'Z')) {
                return nextToken();
            } else if (lastChar < 0x20 || lastChar >= 0x7F) {
                // UTF-8 characters are only allowed inside JSON string literals.
                throw newParseException("Illegal character in JSON structure: '\\u%04x'", lastChar);
            } else {
                return nextSymbol();
            }
        }

        return null;
    }
583321
public boolean next() throws IOException {
		boolean hasNext = true;
		try {
			oldId = stream.readInt();
			oldText = stream.readUTFAsArray();
		} catch (EOFException e) {
			hasNext = false;
		}
		return hasNext;
	}
755357
@Override
    public Sequence<T> nextSequence() {
        Sequence<T> originalSequence = underlyingIterator.nextSequence();
        Sequence<T> newSequence = new Sequence<>();

        if (originalSequence != null)
            for (T element : originalSequence.getElements()) {
                if (element != null && vocabCache.hasToken(element.getLabel())) {
                    newSequence.addElement(vocabCache.wordFor(element.getLabel()));
                }
            }

        newSequence.setSequenceId(originalSequence.getSequenceId());

        return newSequence;
    }