Gets the next token from a tokenizer and converts it to a string.
@return The next token in the stream, as a string.
@throws TextParseException The input was invalid or not a string.
@throws IOException An I/O error occurred.
970096
public String
getString() throws IOException {
	Token next = get();
	if (!next.isString()) {
		throw exception("expected a string");
	}
	return next.value;
}
90689
public int getNextToken() throws java.io.IOException {
  int zzInput;
  int zzAction;

  // cached fields:
  int zzCurrentPosL;
  int zzMarkedPosL;
  int zzEndReadL = zzEndRead;
  char [] zzBufferL = zzBuffer;
  char [] zzCMapL = ZZ_CMAP;

  int [] zzTransL = ZZ_TRANS;
  int [] zzRowMapL = ZZ_ROWMAP;
  int [] zzAttrL = ZZ_ATTRIBUTE;

  while (true) {
    zzMarkedPosL = zzMarkedPos;

    yychar+= zzMarkedPosL-zzStartRead;

    zzAction = -1;

    zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;

    zzState = ZZ_LEXSTATE[zzLexicalState];


    zzForAction: {
      while (true) {
  
        if (zzCurrentPosL < zzEndReadL)
          zzInput = zzBufferL[zzCurrentPosL++];
        else if (zzAtEOF) {
          zzInput = YYEOF;
          break zzForAction;
        }
        else {
          // store back cached positions
          zzCurrentPos  = zzCurrentPosL;
          zzMarkedPos   = zzMarkedPosL;
          boolean eof = zzRefill();
          // get translated positions and possibly new buffer
          zzCurrentPosL  = zzCurrentPos;
          zzMarkedPosL   = zzMarkedPos;
          zzBufferL      = zzBuffer;
          zzEndReadL     = zzEndRead;
          if (eof) {
            zzInput = YYEOF;
            break zzForAction;
          }
          else {
            zzInput = zzBufferL[zzCurrentPosL++];
          }
        }
        int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
        if (zzNext == -1) break zzForAction;
        zzState = zzNext;

        int zzAttributes = zzAttrL[zzState];
        if ( (zzAttributes & 1) == 1 ) {
          zzAction = zzState;
          zzMarkedPosL = zzCurrentPosL;
          if ( (zzAttributes & 8) == 8 ) break zzForAction;
        }

      }
    }

    // store back cached position
    zzMarkedPos = zzMarkedPosL;

    switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
      case 7: 
        { return HOST;
        }
      case 15: break;
      case 6: 
        { return HASHTAG;
        }
      case 16: break;
      case 12: 
        { return ACRONYM;
        }
      case 17: break;
      case 11: 
        { return ACRONYM_DEP;
        }
      case 18: break;
      case 14: 
        { return URL;
        }
      case 19: break;
      case 1: 
        { /* ignore */
        }
      case 20: break;
      case 5: 
        { return MENTION;
        }
      case 21: break;
      case 4: 
        { return PERCENTAGE;
        }
      case 22: break;
      case 8: 
        { return NUM;
        }
      case 23: break;
      case 3: 
        { return CJ;
        }
      case 24: break;
      case 2: 
        { return ALPHANUM;
        }
      case 25: break;
      case 10: 
        { return COMPANY;
        }
      case 26: break;
      case 9: 
        { return APOSTROPHE;
        }
      case 27: break;
      case 13: 
        { return EMAIL;
        }
      case 28: break;
      default: 
        if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
          zzAtEOF = true;
          return YYEOF;
        } 
        else {
          zzScanError(ZZ_NO_MATCH);
        }
    }
  }
}
146667
public Token nextToken() {

        if(lexer._input == null) {
            throw new IllegalStateException("nextToken requires a non-null input stream.");
        } else {
            int tokenStartMarker = lexer._input.mark();

            try {
                Token ttype1;
                label110:
                while(!lexer._hitEOF) {
                    lexer._token = null;
                    lexer._channel = Token.DEFAULT_CHANNEL;
                    lexer._tokenStartCharIndex = lexer._input.index();
                    lexer._tokenStartCharPositionInLine = ((LexerATNSimulator)lexer.getInterpreter()).getCharPositionInLine();
                    lexer._tokenStartLine = ((LexerATNSimulator)lexer.getInterpreter()).getLine();
                    lexer._text = null;


                    do {
                        lexer._type = 0;

                        int ttype;
                        try {
                            ttype = ((LexerATNSimulator)lexer.getInterpreter()).match(lexer._input, lexer._mode);
                        } catch (LexerNoViableAltException var7) {
                            lexer.notifyListeners(var7);
                            lexer.recover(var7);
                            ttype = -3;
                        }

                        if(lexer._input.LA(1) == -1) {
                            lexer._hitEOF = true;
                        }

                        if(lexer._type == 0) {
                            lexer._type = ttype;
                        }

                        if(lexer._type == -3) {
                            continue label110;
                        }
                    } while(lexer._type == -2);

                    if(lexer._token == null) {
                        lexer.emit();
                    }

                    ttype1 = lexer._token;
                    //log.trace("return token: >" + ttype1.getText()+"< " + ttype1.getType());
                    return ttype1;
                }
                // recover from unexpected EOF
                eof = true;
                if (!ls.isBalanced()) {
                    return generateEOFRecover();
                }
                log.trace("lexer state is balanced - emitEOF");
                lexer.emitEOF();
                ttype1 = lexer._token;
                return ttype1;
            } finally {
                lexer._input.release(tokenStartMarker);
            }
        }
    }
90142
public String nextName() throws IOException {
        peek();
        if (token != JsonToken.NAME) {
            throw new IllegalStateException("Expected a name but was " + peek());
        }
        String result = name;
        advance();
        return result;
    }
1359074
@Override
    public Tuple getNext()
            throws IOException {
        Tuple tuple = null;
        try {
            if (isBuildingExample) {
                isBuildingExample = false; // Terminate on the next iteration
                return tupleFactory.newTuple(createPigExample());
            }

            boolean notDone = reader.nextKeyValue();
            if (!notDone) {
                return null;
            }

            ParsedRecord value = reader.getCurrentValue();

            if (value != null) {
                List<Object> values = new ArrayList<>();
                if (onlyWantListOfFields) {
                    values.add(value.getString(ApacheHttpdLogfileRecordReader.FIELDS));
                } else {
                    for (String fieldName : requestedFields) {
                        if (fieldName.endsWith(".*")) {
                            values.add(value.getStringSet(fieldName));
                            continue;
                        } else {
                            EnumSet<Casts> casts = reader.getParser().getCasts(fieldName);

                            if (casts != null) {
                                if (casts.contains(Casts.LONG)) {
                                    values.add(value.getLong(fieldName));
                                    continue;
                                }

                                if (casts.contains(Casts.DOUBLE)) {
                                    values.add(value.getDouble(fieldName));
                                    continue;
                                }
                            }
                        }
                        values.add(value.getString(fieldName));
                    }
                }
                tuple = tupleFactory.newTuple(values);
            }
        } catch (InvalidDissectorException | MissingDissectorsException e) {
            throw new IOException("Fatal error in the parser", e);
        }
        return tuple;
    }
22396
private void getLastToken(StreamTokenizer tokenizer, boolean endOfFileOk) throws IOException, ParseException {
        if ((tokenizer.nextToken() != StreamTokenizer.TT_EOL) && ((tokenizer.ttype != StreamTokenizer.TT_EOF) || !endOfFileOk)) {
            throw new ParseException("end of line expected", tokenizer.lineno());
        }
    }
763002
private void readSparseInstance(StreamTokenizer tokenizer, AttributeDataset data, Attribute[] attributes) throws IOException, ParseException {
        double[] x = responseIndex >= 0 ? new double[attributes.length - 1] : new double[attributes.length];
        double y = Double.NaN;
        int index = -1;
        
        // Get values for all attributes.
        do {
            getNextToken(tokenizer);
            
            // end of instance
            if (tokenizer.ttype == '}') {
                break;
            }
            
            String s = tokenizer.sval.trim();
            if (index < 0) {
                index = Integer.parseInt(s);
                if (index < 0 || index >= attributes.length) {
                    throw new ParseException("Invalid attribute index: " + index, tokenizer.lineno());
                }
                
            } else {
                
                String val = s;
                if (index != responseIndex) {
                    if (val.equals("?")) {
                        x[index] = Double.NaN;
                    } else {
                        x[index] = attributes[index].valueOf(val);
                    }
                } else {
                    if (val.equals("?")) {
                        y = Double.NaN;
                    } else {
                        y = attributes[index].valueOf(val);
                    }
                }
                
                index = -1;
            }
            
        } while (tokenizer.ttype == StreamTokenizer.TT_WORD);
        
        if (Double.isNaN(y)) data.add(x); else data.add(x, y);
    }
533676
public SequenceLabel getNextToken() {
    if (this.nextToken == null) {
      try {
        this.nextToken = this.jlexer.yylex();
      } catch (final IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
      }
    }
    final SequenceLabel result = this.nextToken;
    this.nextToken = null;
    if (result == null) {
      throw new NoSuchElementException();
    }
    return result;
  }
1084467
private final void nextToken()
  {

    if (m_queueMark < m_ops.getTokenQueueSize())
    {
      m_token = (String) m_ops.m_tokenQueue.elementAt(m_queueMark++);
      m_tokenChar = m_token.charAt(0);
    }
    else
    {
      m_token = null;
      m_tokenChar = 0;
    }
  }
886189
public String nextTo(char delimiter) throws JSONException {
		StringBuilder sb = new StringBuilder();
		for (;;) {
			char c = this.next();
			if (c == delimiter || c == 0 || c == '\n' || c == '\r') {
				if (c != 0) {
					this.back();
				}
				return sb.toString().trim();
			}
			sb.append(c);
		}
	}
646124
public Map<String, Object> parseObject() throws IOException {
		Type t = lexer.readNextToken();
		if (t != Type.START_OBJECT) {
			throw new IOException("Unexpected token: " + t);
		}
		
		return parseObjectInternal();
	}
831022
public void tokenise() throws IOException {
		int nextChar = 0;
//		char[] nextChar = new char[1]; // input variable
//		nextChar[0]=0;	// set to 0
		this.setTokeniserState(TOKENISER_NORMAL);	// set initial tokeniser state
		
		
//		while(this.pbReader.read(nextChar) != -1) {
		while((nextChar = this.pbReader.read()) != -1) {
			this.byteCount++;
			
	        if (this.getTokeniserState() == TOKENISER_BINARY)                      // if we're parsing binary data, handle it directly
	        {
	            if (parseChar(nextChar) != errOK)
	                return; 
	        }  else {
//				switch(nextChar[0]) {
				switch(nextChar) {
					case '{':	// scope delimiter - Open
						this.handleOpenGroup();
						break;
					case '}':  // scope delimiter - Close
						this.handleCloseGroup();
						break;
					case 0x0a:	// noise character
					case 0x0d:	// noise character
//						if(this.isImport()) {
//							this.rtfDoc.add(new RtfDirectContent(new String(nextChar)));
//						}
						break;
					case '\\':	// Control word start delimiter
							if(parseCtrlWord(pbReader) != errOK) {
							// TODO: Indicate some type of error
							return;
						}
						break;
					default:
						if(groupLevel == 0) { // BOMs
							break;
						}
						if(this.getTokeniserState() == TOKENISER_HEX) {
							StringBuffer hexChars = new StringBuffer();
							hexChars.append(nextChar);
//							if(pbReader.read(nextChar) == -1) {
							if((nextChar = pbReader.read()) == -1) {
								return;
							}
							this.byteCount++;
							hexChars.append(nextChar);
	                    	try {
//								nextChar[0]=(char)Integer.parseInt(hexChars.toString(), 16);
								nextChar=Integer.parseInt(hexChars.toString(), 16);
							} catch (NumberFormatException e) {
								return;
							}
		                    this.setTokeniserState(TOKENISER_NORMAL);
						}
						if (parseChar(nextChar) != errOK) {
							// some error occurred. we should send a real error
							return; 							
						}
						break;
				}	// switch(nextChar[0])
			}	// end if (this.getTokeniserState() == TOKENISER_BINARY)
	        
//	        if(groupLevel < 1 && this.isImportFragment()) return; //return errOK;
//	        if(groupLevel < 0 && this.isImportFull()) return; //return errStackUnderflow;
//	        if(groupLevel < 0 && this.isConvert()) return; //return errStackUnderflow;
	        
		}// end while(reader.read(nextChar) != -1)
		RtfDestination dest = this.getCurrentDestination();
		if(dest != null) {
			dest.closeDestination();
		}
	}
1167337
public T getNext() throws IOException {
    try {
      return mapper.readValue(jsonParser, clazz);
    } catch (EOFException e) {
      return null;
    }
  }
279924
@Override
    public final boolean incrementToken() throws IOException
    {
        clearAttributes();
        int posIncr = 1;
        if(tokensList.size()>0)
        {
            this.restoreState(tokensList.remove());
            return true;
        }
        while (true)
        {
            int tokenType = scanner.getNextToken();

            if (tokenType == TweetTokenizerImpl.YYEOF)
            {
                return false;
            }

            if (scanner.yylength() <= maxTokenLength)
            {
                posIncrAtt.setPositionIncrement(posIncr);
                scanner.getText(termAtt);
                final int start = scanner.yychar();
                offsetAtt.setOffset(correctOffset(start), correctOffset(start
                        + termAtt.length()));
                // This 'if' should be removed in the next release. For now, it
                // converts
                // invalid acronyms to HOST. When removed, only the 'else' part
                // should
                // remain.
                if (tokenType == TweetTokenizerImpl.ACRONYM_DEP)
                {
                    if (replaceInvalidAcronym)
                    {
                        typeAtt.setType(TweetTokenizerImpl.TOKEN_TYPES[TweetTokenizerImpl.HOST]);
                        termAtt.setLength(termAtt.length() - 1); // remove
                                                                         // extra
                                                                         // '.'
                    }
                    else
                    {
                        typeAtt.setType(TweetTokenizerImpl.TOKEN_TYPES[TweetTokenizerImpl.ACRONYM]);
                    }
                }
                else
                {
                    typeAtt.setType(TweetTokenizerImpl.TOKEN_TYPES[tokenType]);
                }
                if(tokensList.size()>0)
                    this.restoreState(tokensList.remove());
                /**
                 * End
                 */
                return true;
            }
            else
                // When we skip a too-long term, we still increment the
                // position increment
                posIncr++;
        }
    }
469467
private int peekToken()
        throws IOException
    {
        // By far the most common case:  last token hasn't been consumed,
        // so return already-peeked token.
        if (currentFlaggedToken != Token.EOF) {
            return currentToken;
        }

        int lineno = ts.getLineno();
        int tt = ts.getToken();
        boolean sawEOL = false;

        // process comments and whitespace
        while (tt == Token.EOL || tt == Token.COMMENT) {
            if (tt == Token.EOL) {
                lineno++;
                sawEOL = true;
                tt = ts.getToken();
            } else {
                if (compilerEnv.isRecordingComments()) {
                    String comment = ts.getAndResetCurrentComment();
                    recordComment(lineno, comment);
                    // Comments may contain multiple lines, get the number
                    // of EoLs and increase the lineno
                    lineno += getNumberOfEols(comment);
                    break;
                }tt = ts.getToken();
            }
        }

        currentToken = tt;
        currentFlaggedToken = tt | (sawEOL ? TI_AFTER_EOL : 0);
        return currentToken;  // return unflagged token
    }
289351
public Token next() throws IOException {

        // skip whitespace
        while (Character.isWhitespace((char)lastChar))
        {
            readChar();
        }

        // handle punctuation
        switch (lastChar)
        {
            case -1:  readChar(); return Token.TokenEOF;
            case '{': readChar(); return Token.TokenBraceL;
            case '}': readChar(); return Token.TokenBraceR;
            case '[': readChar(); return Token.TokenBrackL;
            case ']': readChar(); return Token.TokenBrackR;
            case ':': readChar(); return Token.TokenColon;
            case ',': readChar(); return Token.TokenComma;

            case '"':
            case '\'':
                String stringValue = readString();
                return new Token(stringValue);

            case '-':
            case '.':
            case '0':
            case '1':
            case '2':
            case '3':
            case '4':
            case '5':
            case '6':
            case '7':
            case '8':
            case '9':
                Number numberValue = readNumber();
                return new Token(numberValue);

            case 'n': 
            case 't':
            case 'f':
                String ident = readIdentifier();

                if (ident.equals("null"))  return Token.TokenNull;
                if (ident.equals("true"))  return Token.TokenTrue;
                if (ident.equals("false")) return Token.TokenFalse;

                throw new IOException("Unexpected identifier '" + ident + "' " + onLineCol());

            default:
                throw new IOException("Unexpected character '" + (char)lastChar + "' " + onLineCol());

        }

    }
170424
@Override
    public String nextToken()
        throws NoSuchElementException
    {
        if (!hasMoreTokens() || _token==null)
            throw new NoSuchElementException();
        String t=_token.toString();
        _token.setLength(0);
        _hasToken=false;
        return t;
    }