This method returns next token from prebuffered list of tokens or underlying InputStream

@return next token as String
494689
@Override
    public String nextToken() {
        if (!tokens.isEmpty() && position.get() < tokens.size())
            return tokens.get(position.getAndIncrement());

        return nextTokenFromStream();
    }
1163510
@Nullable
    public Token next() throws IOException {
        if (unreadToken != null) {
            lastToken = unreadToken;
            unreadToken = null;
            return lastToken;
        }

        while (lastChar >= 0) {
            if (lastChar == 0) {
                if (!readNextChar()) {
                    break;
                }
            }
            if (lastChar == Token.kNewLine ||
                lastChar == Token.kCarriageReturn ||
                lastChar == Token.kSpace ||
                lastChar == Token.kTab) {
                lastChar = 0;
                continue;
            }

            if (lastChar == Token.kShellComment) {
                getRestOfLine();
                continue;
            }

            if (lastChar == Token.kLiteralDoubleQuote || lastChar == Token.kLiteralQuote) {
                lastToken = nextStringLiteral((char) lastChar);
                return lastToken;
            } else if (lastChar == '.' || lastChar == '-' || (lastChar >= '0' && lastChar <= '9')) {
                lastToken = nextNumber();
                return lastToken;
            } else if ('_' == lastChar ||
                       ('a' <= lastChar && lastChar <= 'z') ||
                       ('A' <= lastChar && lastChar <= 'Z')) {
                lastToken = nextToken();
                return lastToken;
            } else if (lastChar < 0x20 || lastChar >= 0x7F) {
                // non-ASCII UTF-8 characters are only allowed inside JSON string literals.
                throw failure(lineNo, linePos, 1, "Unknown token initiator '%s'", Strings.escape((char) lastChar));
            } else {
                lastToken = nextSymbol();
                return lastToken;
            }
        }

        return null;
    }
90689
public int getNextToken() throws java.io.IOException {
  int zzInput;
  int zzAction;

  // cached fields:
  int zzCurrentPosL;
  int zzMarkedPosL;
  int zzEndReadL = zzEndRead;
  char [] zzBufferL = zzBuffer;
  char [] zzCMapL = ZZ_CMAP;

  int [] zzTransL = ZZ_TRANS;
  int [] zzRowMapL = ZZ_ROWMAP;
  int [] zzAttrL = ZZ_ATTRIBUTE;

  while (true) {
    zzMarkedPosL = zzMarkedPos;

    yychar+= zzMarkedPosL-zzStartRead;

    zzAction = -1;

    zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;

    zzState = ZZ_LEXSTATE[zzLexicalState];


    zzForAction: {
      while (true) {
  
        if (zzCurrentPosL < zzEndReadL)
          zzInput = zzBufferL[zzCurrentPosL++];
        else if (zzAtEOF) {
          zzInput = YYEOF;
          break zzForAction;
        }
        else {
          // store back cached positions
          zzCurrentPos  = zzCurrentPosL;
          zzMarkedPos   = zzMarkedPosL;
          boolean eof = zzRefill();
          // get translated positions and possibly new buffer
          zzCurrentPosL  = zzCurrentPos;
          zzMarkedPosL   = zzMarkedPos;
          zzBufferL      = zzBuffer;
          zzEndReadL     = zzEndRead;
          if (eof) {
            zzInput = YYEOF;
            break zzForAction;
          }
          else {
            zzInput = zzBufferL[zzCurrentPosL++];
          }
        }
        int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
        if (zzNext == -1) break zzForAction;
        zzState = zzNext;

        int zzAttributes = zzAttrL[zzState];
        if ( (zzAttributes & 1) == 1 ) {
          zzAction = zzState;
          zzMarkedPosL = zzCurrentPosL;
          if ( (zzAttributes & 8) == 8 ) break zzForAction;
        }

      }
    }

    // store back cached position
    zzMarkedPos = zzMarkedPosL;

    switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
      case 7: 
        { return HOST;
        }
      case 15: break;
      case 6: 
        { return HASHTAG;
        }
      case 16: break;
      case 12: 
        { return ACRONYM;
        }
      case 17: break;
      case 11: 
        { return ACRONYM_DEP;
        }
      case 18: break;
      case 14: 
        { return URL;
        }
      case 19: break;
      case 1: 
        { /* ignore */
        }
      case 20: break;
      case 5: 
        { return MENTION;
        }
      case 21: break;
      case 4: 
        { return PERCENTAGE;
        }
      case 22: break;
      case 8: 
        { return NUM;
        }
      case 23: break;
      case 3: 
        { return CJ;
        }
      case 24: break;
      case 2: 
        { return ALPHANUM;
        }
      case 25: break;
      case 10: 
        { return COMPANY;
        }
      case 26: break;
      case 9: 
        { return APOSTROPHE;
        }
      case 27: break;
      case 13: 
        { return EMAIL;
        }
      case 28: break;
      default: 
        if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
          zzAtEOF = true;
          return YYEOF;
        } 
        else {
          zzScanError(ZZ_NO_MATCH);
        }
    }
  }
}
984745
public String getContent() {
    	StringBuffer buf = new StringBuffer();
    	for (Iterator i = getChunks().iterator(); i.hasNext(); ) {
    		buf.append(i.next().toString());
    	}
    	return buf.toString();
    }
874148
def next_token(self):
        """"""
        if self.lookahead:
            self.current_token = self.lookahead.popleft()
            return self.current_token

        self.current_token = self._parse_next_token()
        return self.current_token
649540
private String
remainingStrings() throws IOException {
        StringBuffer buffer = null;
        while (true) {
                Tokenizer.Token t = get();
                if (!t.isString())
                        break;
                if (buffer == null)
                        buffer = new StringBuffer();
                buffer.append(t.value);
        }
        unget();
        if (buffer == null)
                return null;
        return buffer.toString();
}
531574
public String nextToken() 
    {
        if (m_tokenOffset_ < 0) {
            if (m_nextOffset_ < 0) {
                throw new NoSuchElementException("No more tokens in String");   
            }
            // pre-calculations of tokens not done
            if (m_returnDelimiters_) {
                int tokenlimit = 0;
                int c = UTF16.charAt(m_source_, m_nextOffset_);
                boolean contains = delims == null 
                    ? m_delimiters_.contains(c) 
                    : c < delims.length && delims[c];
                if (contains) {
                     if (m_coalesceDelimiters_) {
                        tokenlimit = getNextNonDelimiter(m_nextOffset_);
                     } else {
                        tokenlimit = m_nextOffset_ + UTF16.getCharCount(c);
                        if (tokenlimit == m_length_) {
                            tokenlimit = -1;
                        }
                     }
                }
                else {
                    tokenlimit = getNextDelimiter(m_nextOffset_);
                }
                String result;
                if (tokenlimit < 0) {
                    result = m_source_.substring(m_nextOffset_);
                }
                else {
                    result = m_source_.substring(m_nextOffset_, tokenlimit);
                }
                m_nextOffset_ = tokenlimit;
                return result;
            }
            else {
                int tokenlimit = getNextDelimiter(m_nextOffset_);
                String result;
                if (tokenlimit < 0) {
                    result = m_source_.substring(m_nextOffset_);
                    m_nextOffset_ = tokenlimit;
                }
                else {
                    result = m_source_.substring(m_nextOffset_, tokenlimit);
                    m_nextOffset_ = getNextNonDelimiter(tokenlimit);
                }
                
                return result;
            }
        }
        // count was called before and we have all the tokens
        if (m_tokenOffset_ >= m_tokenSize_) {
            throw new NoSuchElementException("No more tokens in String");
        }
        String result;
        if (m_tokenLimit_[m_tokenOffset_] >= 0) {
            result = m_source_.substring(m_tokenStart_[m_tokenOffset_],
                                         m_tokenLimit_[m_tokenOffset_]);
        }
        else {
            result = m_source_.substring(m_tokenStart_[m_tokenOffset_]);
        }
        m_tokenOffset_ ++;
        m_nextOffset_ = -1;
        if (m_tokenOffset_ < m_tokenSize_) {
            m_nextOffset_ = m_tokenStart_[m_tokenOffset_];
        }
        return result;
    }
146667
public Token nextToken() {

        if(lexer._input == null) {
            throw new IllegalStateException("nextToken requires a non-null input stream.");
        } else {
            int tokenStartMarker = lexer._input.mark();

            try {
                Token ttype1;
                label110:
                while(!lexer._hitEOF) {
                    lexer._token = null;
                    lexer._channel = Token.DEFAULT_CHANNEL;
                    lexer._tokenStartCharIndex = lexer._input.index();
                    lexer._tokenStartCharPositionInLine = ((LexerATNSimulator)lexer.getInterpreter()).getCharPositionInLine();
                    lexer._tokenStartLine = ((LexerATNSimulator)lexer.getInterpreter()).getLine();
                    lexer._text = null;


                    do {
                        lexer._type = 0;

                        int ttype;
                        try {
                            ttype = ((LexerATNSimulator)lexer.getInterpreter()).match(lexer._input, lexer._mode);
                        } catch (LexerNoViableAltException var7) {
                            lexer.notifyListeners(var7);
                            lexer.recover(var7);
                            ttype = -3;
                        }

                        if(lexer._input.LA(1) == -1) {
                            lexer._hitEOF = true;
                        }

                        if(lexer._type == 0) {
                            lexer._type = ttype;
                        }

                        if(lexer._type == -3) {
                            continue label110;
                        }
                    } while(lexer._type == -2);

                    if(lexer._token == null) {
                        lexer.emit();
                    }

                    ttype1 = lexer._token;
                    //log.trace("return token: >" + ttype1.getText()+"< " + ttype1.getType());
                    return ttype1;
                }
                // recover from unexpected EOF
                eof = true;
                if (!ls.isBalanced()) {
                    return generateEOFRecover();
                }
                log.trace("lexer state is balanced - emitEOF");
                lexer.emitEOF();
                ttype1 = lexer._token;
                return ttype1;
            } finally {
                lexer._input.release(tokenStartMarker);
            }
        }
    }
150166
private int next() throws IOException {
        int c = get();
        if (c == '/') {
            switch (peek()) {
                case '/':
                    for (; ; ) {
                        c = get();
                        if (c <= '\n') {
                            break;
                        }
                    }
                    break;
                case '*':
                    get();
                    while (c != ' ') {
                        switch (get()) {
                            case '*':
                                if (peek() == '/') {
                                    get();
                                    c = ' ';
                                }
                                break;
                            case EOF:
                                error("Unterminated comment.");
                        }
                    }
                    break;
            }
        }
        theY = theX;
        theX = c;
        return c;
    }
1318153
public final String[] getTokensWithMultiWords(final String[] tokens) {
    final Span[] multiWordSpans = multiWordsToSpans(tokens);
    final List<String> tokenList = new ArrayList<String>(Arrays.asList(tokens));
    int counter = 0;
    for (final Span mwSpan : multiWordSpans) {
      final int fromIndex = mwSpan.getStart() - counter;
      final int toIndex = mwSpan.getEnd() - counter;
      // System.err.println(fromIndex + " " + toIndex);
      // add to the counter the length of the sublist removed
      // to allow the fromIndex and toIndex to match wrt to the tokenList
      // indexes
      counter = counter + tokenList.subList(fromIndex, toIndex).size() - 1;
      // create the multiword joining the sublist
      final String multiWord = Joiner.on("#").join(
          tokenList.subList(fromIndex, toIndex));
      // remove the sublist containing the tokens to be replaced in the span
      tokenList.subList(fromIndex, toIndex).clear();
      // add the multiword containing the tokens in one Span
      tokenList.add(fromIndex, multiWord);
    }
    return tokenList.toArray(new String[tokenList.size()]);
  }
277812
private HttpTokens.Token next(ByteBuffer buffer) {
        byte ch = buffer.get();

        HttpTokens.Token t = HttpTokens.TOKENS[0xff & ch];

        if (DEBUG)
            LOG.debug("token={}", t);

        switch (t.getType()) {
            case CNTL:
                throw new IllegalCharacterException(_state, t, buffer);

            case LF:
                _cr = false;
                break;

            case CR:
                if (_cr)
                    throw new BadMessageException("Bad EOL");

                _cr = true;
                return null;

            case ALPHA:
            case DIGIT:
            case TCHAR:
            case VCHAR:
            case HTAB:
            case SPACE:
            case OTEXT:
            case COLON:
                if (_cr)
                    throw new BadMessageException("Bad EOL");
                break;

            default:
                break;
        }

        return t;
    }
821619
public String nextString() {
        char[] buffer = new char[length];
        for (int index = 0; index < length; index++) {
            buffer[index] = SYMBOL[random.nextInt(SYMBOL.length)];
        }
        return new String(buffer);
    }
829773
public T next() {
    if (nextToken == null) {
      nextToken = getNext();
    }
    T result = nextToken;
    nextToken = null;
    if (result == null) {
      throw new NoSuchElementException();
    }
    return result;
  }
170424
@Override
    public String nextToken()
        throws NoSuchElementException
    {
        if (!hasMoreTokens() || _token==null)
            throw new NoSuchElementException();
        String t=_token.toString();
        _token.setLength(0);
        _hasToken=false;
        return t;
    }
353856
private String getTokenOrBlank(final List<String> tokens, final int reverseIndex) {

		final int length = tokens.size();
		final int index  = length - reverseIndex - 1;

		if (index >= 0 && length > 0 && index < length) {
			return tokens.get(index);
		}

		return "";
	}
808895
public final synchronized String next() {
        // Increment and capture.
        seq += inc;
        if (seq >= maxSeq) {
            randomizePrefix();
            resetSequential();
        }

        // Copy prefix
        char[] b = new char[totalLen];
        System.arraycopy(pre, 0, b, 0, preLen);

        // copy in the seq in base36.
        int i = b.length;
        for (long l = seq; i > preLen; l /= base) {
            i--;
            b[i] = digits[(int) (l % base)];
        }
        return new String(b);
    }
968186
final String toTokenString() throws IOException {
    String[] stringToFormat = toFinalString(null);
    return String.join("|", Arrays.asList(stringToFormat));
  }