Returns the next literal string or token/delimiter.
198783
public Object next()
    {
        String result = null;

        if ( _literal != null )
        {
            result = _literal;
            _literal = null;
        }
        else if ( _token != null )
        {
            result = _token;
            _token = null;
        }
        return result;
    }
146255
public Token scanNumericLiteral() {
		int start = pos;
		int next = pos + 1;
		// Decide whether it's an integer, binary or hexadecimal literal
		if (next < input.length() && input.charAt(pos) == '0' && input.charAt(next) == 'x') {
			// Hexadecimal literal
			return scanHexLiteral();
		} else if (next < input.length() && input.charAt(pos) == '0' && input.charAt(next) == 'b') {
			// Binary literal
			return scanBinaryLiteral();
		} else {
			return scanIntegerLiteral();
		}
	}
531574
public String nextToken() 
    {
        if (m_tokenOffset_ < 0) {
            if (m_nextOffset_ < 0) {
                throw new NoSuchElementException("No more tokens in String");   
            }
            // pre-calculations of tokens not done
            if (m_returnDelimiters_) {
                int tokenlimit = 0;
                int c = UTF16.charAt(m_source_, m_nextOffset_);
                boolean contains = delims == null 
                    ? m_delimiters_.contains(c) 
                    : c < delims.length && delims[c];
                if (contains) {
                     if (m_coalesceDelimiters_) {
                        tokenlimit = getNextNonDelimiter(m_nextOffset_);
                     } else {
                        tokenlimit = m_nextOffset_ + UTF16.getCharCount(c);
                        if (tokenlimit == m_length_) {
                            tokenlimit = -1;
                        }
                     }
                }
                else {
                    tokenlimit = getNextDelimiter(m_nextOffset_);
                }
                String result;
                if (tokenlimit < 0) {
                    result = m_source_.substring(m_nextOffset_);
                }
                else {
                    result = m_source_.substring(m_nextOffset_, tokenlimit);
                }
                m_nextOffset_ = tokenlimit;
                return result;
            }
            else {
                int tokenlimit = getNextDelimiter(m_nextOffset_);
                String result;
                if (tokenlimit < 0) {
                    result = m_source_.substring(m_nextOffset_);
                    m_nextOffset_ = tokenlimit;
                }
                else {
                    result = m_source_.substring(m_nextOffset_, tokenlimit);
                    m_nextOffset_ = getNextNonDelimiter(tokenlimit);
                }
                
                return result;
            }
        }
        // count was called before and we have all the tokens
        if (m_tokenOffset_ >= m_tokenSize_) {
            throw new NoSuchElementException("No more tokens in String");
        }
        String result;
        if (m_tokenLimit_[m_tokenOffset_] >= 0) {
            result = m_source_.substring(m_tokenStart_[m_tokenOffset_],
                                         m_tokenLimit_[m_tokenOffset_]);
        }
        else {
            result = m_source_.substring(m_tokenStart_[m_tokenOffset_]);
        }
        m_tokenOffset_ ++;
        m_nextOffset_ = -1;
        if (m_tokenOffset_ < m_tokenSize_) {
            m_nextOffset_ = m_tokenStart_[m_tokenOffset_];
        }
        return result;
    }
278399
@Override
	public String next() {
		Long current = next;
		next += 1;

		return String.format(format, current);
	}
384312
public String getText() {
        StringBuilder buf = new StringBuilder();
        boolean paste = false;
        for (Token tok : tokens) {
            if (tok.getType() == Token.M_PASTE) {
                assert paste == false : "Two sequential pastes.";
                paste = true;
                continue;
            } else {
                buf.append(tok.getText());
            }
            if (paste) {
                buf.append(" #" + "# ");
                paste = false;
            }
            // buf.append(tokens.get(i));
        }
        return buf.toString();
    }
388690
def nextto(regex)
      buf = ""
      while (true)
	c = self.nextchar()
	if !(regex =~ c).nil? || c == '\0' || c == '\n' || c == '\r'
	  self.back() if (c != '\0')
	  return(buf.chomp())
	end
	buf += c
      end
    end
1150506
def next_token(self):
        """"""
        while self.char is not None:

            if self.char.isspace():
                # The current character is a whitespace
                self.whitespace()
                continue

            elif self.char == '#':
                # The current character is `#`
                self.advance()
                self.comment()
                continue

            elif self.char.isalpha() or self.char == '_':
                # The current character is a letter or `_`
                return self._id()

            elif self.char == ';':
                # The current character is `;`
                self.advance()
                return Token(Nature.SEMI, ';')

            elif self.char == ',':
                # The current character is `,`
                self.advance()
                return Token(Nature.COMMA, ';')

            elif self.char.isdigit():
                # The current character is a number
                return self.number()

            elif self.char == '=' and self.peek() == '=':
                # The current character is `==`
                self.advance()
                self.advance()
                return Token(Nature.EQ, '==')

            elif self.char == '!' and self.peek() == '=':
                # The current character is `!=`
                self.advance()
                self.advance()
                return Token(Nature.NE, '!=')

            elif self.char == '<' and self.peek() == '=':
                # The current character is `<=`
                self.advance()
                self.advance()
                return Token(Nature.LE, '<=')

            elif self.char == '>' and self.peek() == '=':
                # The current character is `>=`
                self.advance()
                self.advance()
                return Token(Nature.GE, '>=')

            elif self.char == '<':
                # The current character is `<`
                self.advance()
                return Token(Nature.LT, '<')

            elif self.char == '>':
                # The current character is `>`
                self.advance()
                return Token(Nature.GT, '>')

            elif self.char == '=':
                # The current character is `=`
                self.advance()
                return Token(Nature.ASSIGN, '=')

            elif self.char == '+':
                # The current character is `+`
                self.advance()
                return Token(Nature.PLUS, '+')

            elif self.char == '-':
                # The current character is `-`
                self.advance()
                return Token(Nature.MINUS, '-')

            elif self.char == '*':
                # The current character is `*`
                self.advance()
                return Token(Nature.MUL, '*')

            elif self.char == '/' and self.peek() == '/':
                # The current character is `//`
                self.advance()
                self.advance()
                return Token(Nature.INT_DIV, '//')

            elif self.char == '/':
                # The current character is `/`
                self.advance()
                return Token(Nature.DIV, '/')

            elif self.char == '(':
                # The current character is `(`
                self.advance()
                return Token(Nature.LPAREN, '(')

            elif self.char == ')':
                # The current character is `)`
                self.advance()
                return Token(Nature.RPAREN, ')')

            elif self.char == '{':
                # The current character is `{`
                self.advance()
                return Token(Nature.LBRACKET, '{')

            elif self.char == '}':
                # The current character is `}`
                self.advance()
                return Token(Nature.RBRACKET, '}')

            else:
                # The current character is unknown
                raise LexicalError(f"Invalid character `{self.char}`.")

        # End of raw input
        return Token(Nature.EOF, None)
1040991
def _next(self):
        
        c = self._get()
        if c == '/':
            p = self._peek()
            if p == '/':
                c = self._get()
                while c > '\n':
                    c = self._get()
                return c
            if p == '*':
                c = self._get()
                while 1:
                    c = self._get()
                    if c == '*':
                        if self._peek() == '/':
                            self._get()
                            return ' '
                    if c == '\000':
                        raise UnterminatedComment()

        return c
576118
def NextToken(self):
    """"""
    # Nothing in the input stream - no token can match.
    if not self.buffer:
      return

    current_state = self.state
    for token in self._tokens:
      # Does the rule apply to us?
      if token.state_regex and not token.state_regex.match(current_state):
        continue

      if self.verbose:
        logging.debug("%s: Trying to match %r with %r", self.state,
                      self.buffer[:10], token.re_str)

      # Try to match the rule
      m = token.regex.match(self.buffer)
      if not m:
        continue

      if self.verbose:
        logging.debug("%s matched %s", token.re_str, m.group(0))

      # A token matched the empty string. We can not consume the token from the
      # input stream.
      if m.end() == 0:
        raise RuntimeError("Lexer bug! Token can not match the empty string.")

      # The match consumes the data off the buffer (the handler can put it back
      # if it likes)
      self.processed_buffer += self.buffer[:m.end()]
      self.buffer = self.buffer[m.end():]
      self.processed += m.end()

      next_state = token.next_state
      for action in token.actions:
        if self.verbose:
          logging.debug("Calling %s with %s", action, m.group(0))

        # Is there a callback to handle this action?
        cb = getattr(self, action, self.Default)

        # Allow a callback to skip other callbacks.
        try:
          possible_next_state = cb(string=m.group(0), match=m)
          if possible_next_state == "CONTINUE":
            continue
          # Override the state from the Token
          elif possible_next_state:
            next_state = possible_next_state
        except ParseError as e:
          self.Error(e)

      # Update the next state
      if next_state:
        self.state = next_state

      return token

    # Check that we are making progress - if we are too full, we assume we are
    # stuck.
    self.Error("Lexer stuck at state %s" % (self.state))
    self.processed_buffer += self.buffer[:1]
    self.buffer = self.buffer[1:]
    return "Error"
20481
public function NextToken()
	{
		$anchor = $this->_offset;
		$ch = '\0';
		$this->Value = "";
		while ( $this->_offset < strlen( $this->_text ) )
		{
			$ch = $this->_text[ $this->_offset ];
			if ( preg_match( "/\s/", $ch ) && $ch != '\n' && $ch != '\r' )
				$this->_offset++;
			else
				break;
		}
		switch ( $ch )
		{
			case '\0':

				$this->Token = $ch;
				break;

			case '\n':

				$this->_offset++;
				if ($this->_offset < strlen( $this->_text ) - 1 && $this->_text[ $this->_offset ] == '\r' )
					$this->_offset++;
				$this->LineCount++;
				$this->Token = $ch;
				break;

			case '\r':

				$this->_offset++;
				if ( $this->_offset < strlen( $this->_text.Length ) - 1 && $this->_text[ $this->_offset ] == '\n')
					$this->_offset++;
				$this->LineCount++;
				$this->Token = '\n';
				break;

			default:

				if ( is_numeric( $ch ) )
				{
					$sb = array();
					while ( $this->_offset < strlen( $this->_text ) )
					{
						$ch = $this->_text[ $this->_offset ];
						if ( ! is_numeric( $ch ) )
							break;

						$sb[] = $ch;
						$this->_offset++;
					}
					$this->Token = StringTokenizer::TokenInt;
					$this->Value = implode( "", $sb );
				}
				else
				{
					$this->Token = $ch;
					$this->_offset++;
				}
				break;

		}
		return $this->Token;
	}
692596
protected function _getNextToken()
    {
        $this->token      = self::EOF;
        $this->tokenValue = null;
        $this->_eatWhitespace();

        if ($this->offset >= $this->sourceLength) {
            return(self::EOF);
        }

        $str       = $this->source;
        $strLength = $this->sourceLength;
        $i         = $this->offset;
        $start     = $i;

        switch ($str{$i}) {
            case '{':
                $this->token = self::LBRACE;
                break;
            case '}':
                $this->token = self::RBRACE;
                break;
            case '[':
                $this->token = self::LBRACKET;
                break;
            case ']':
                $this->token = self::RBRACKET;
                break;
            case ',':
                $this->token = self::COMMA;
                break;
            case ':':
                $this->token = self::COLON;
                break;
            case '"':
                $result = '';
                do {
                    $i++;
                    if ($i >= $strLength) {
                        break;
                    }

                    $chr = $str{$i};

                    if ($chr == '\\') {
                        $i++;
                        if ($i >= $strLength) {
                            break;
                        }
                        $chr = $str{$i};
                        switch ($chr) {
                            case '"':
                                $result .= '"';
                                break;
                            case '\\':
                                $result .= '\\';
                                break;
                            case '/':
                                $result .= '/';
                                break;
                            case 'b':
                                $result .= "\x08";
                                break;
                            case 'f':
                                $result .= "\x0c";
                                break;
                            case 'n':
                                $result .= "\x0a";
                                break;
                            case 'r':
                                $result .= "\x0d";
                                break;
                            case 't':
                                $result .= "\x09";
                                break;
                            case '\'':
                                $result .= '\'';
                                break;
                            default:
                                throw new RuntimeException("Illegal escape sequence '{$chr}'");
                        }
                    } elseif ($chr == '"') {
                        break;
                    } else {
                        $result .= $chr;
                    }
                } while ($i < $strLength);

                $this->token = self::DATUM;
                //$this->tokenValue = substr($str, $start + 1, $i - $start - 1);
                $this->tokenValue = $result;
                break;
            case 't':
                if (($i+ 3) < $strLength && substr($str, $start, 4) == "true") {
                    $this->token = self::DATUM;
                }
                $this->tokenValue = true;
                $i += 3;
                break;
            case 'f':
                if (($i+ 4) < $strLength && substr($str, $start, 5) == "false") {
                    $this->token = self::DATUM;
                }
                $this->tokenValue = false;
                $i += 4;
                break;
            case 'n':
                if (($i+ 3) < $strLength && substr($str, $start, 4) == "null") {
                    $this->token = self::DATUM;
                }
                $this->tokenValue = null;
                $i += 3;
                break;
        }

        if ($this->token != self::EOF) {
            $this->offset = $i + 1; // Consume the last token character
            return($this->token);
        }

        $chr = $str{$i};
        if ($chr == '-' || $chr == '.' || ($chr >= '0' && $chr <= '9')) {
            if (preg_match('/-?([0-9])*(\.[0-9]*)?((e|E)((-|\+)?)[0-9]+)?/s', $str, $matches, PREG_OFFSET_CAPTURE, $start) && $matches[0][1] == $start) {
                $datum = $matches[0][0];

                if (is_numeric($datum)) {
                    if (preg_match('/^0\d+$/', $datum)) {
                        throw new RuntimeException("Octal notation not supported by JSON (value: {$datum})");
                    } else {
                        $val  = intval($datum);
                        $fVal = floatval($datum);
                        $this->tokenValue = ($val == $fVal ? $val : $fVal);
                    }
                } else {
                    throw new RuntimeException("Illegal number format: {$datum}");
                }

                $this->token = self::DATUM;
                $this->offset = $start + strlen($datum);
            }
        } else {
            throw new RuntimeException('Illegal Token');
        }

        return $this->token;
    }
1161151
@SuppressWarnings("fallthrough")
  private String nextUnquotedValue()
  {
    int i = 0;

    findNonLiteralCharacter:
    for (; pos + i < limit; i++) {
      switch (in.charAt(pos + i)) {
      case '/':
      case '\\':
      case ';':
      case '#':
      case '=':
        checkLenient(); // fall-through
      case '{':
      case '}':
      case '[':
      case ']':
      case ':':
      case ',':
      case ' ':
      case '\t':
      case '\f':
      case '\r':
      case '\n':
        break findNonLiteralCharacter;
      }
    }

    String result = in.substring( pos, pos + i);
    pos += i;
    return result;
  }
744953
protected function _getNextToken()
    {
        $this->_token      = self::EOF;
        $this->_tokenValue = null;
        $this->_eatWhitespace();

        if ($this->_offset >= $this->_sourceLength) {
            return(self::EOF);
        }

        $str        = $this->_source;
        $str_length = $this->_sourceLength;
        $i          = $this->_offset;
        $start      = $i;

        switch ($str{$i}) {
        case '{':
            $this->_token = self::LBRACE;
            break;
        case '}':
            $this->_token = self::RBRACE;
            break;
        case '[':
            $this->_token = self::LBRACKET;
            break;
        case ']':
            $this->_token = self::RBRACKET;
            break;
        case ',':
            $this->_token = self::COMMA;
            break;
        case ':':
            $this->_token = self::COLON;
            break;
        case  '"':
            $result = '';
            do {
                $i++;
                if ($i >= $str_length) {
                    break;
                }

                $chr = $str{$i};
                if ($chr == '\\') {
                    $i++;
                    if ($i >= $str_length) {
                        break;
                    }
                    $chr = $str{$i};
                    switch ($chr) {
                    case '"' :
                        $result .= '"';
                        break;
                    case '\\':
                        $result .= '\\';
                        break;
                    case '/' :
                        $result .= '/';
                        break;
                    case 'b' :
                        $result .= chr(8);
                        break;
                    case 'f' :
                        $result .= chr(12);
                        break;
                    case 'n' :
                        $result .= chr(10);
                        break;
                    case 'r' :
                        $result .= chr(13);
                        break;
                    case 't' :
                        $result .= chr(9);
                        break;
                    case '\'' :
                        $result .= '\'';
                        break;
                    default:
                        throw new Zend_Json_Exception(
                            "Illegal escape "
                                .  "sequence '" . $chr . "'"
                        );
                    }
                } elseif ($chr == '"') {
                    break;
                } else {
                    $result .= $chr;
                }
            } while ($i < $str_length);

            $this->_token = self::DATUM;
            //$this->_tokenValue = substr($str, $start + 1, $i - $start - 1);
            $this->_tokenValue = $result;
            break;
        case 't':
            if (($i+ 3) < $str_length && substr($str, $start, 4) == "true") {
                $this->_token = self::DATUM;
            }
            $this->_tokenValue = true;
            $i += 3;
            break;
        case 'f':
            if (($i+ 4) < $str_length && substr($str, $start, 5) == "false") {
                $this->_token = self::DATUM;
            }
            $this->_tokenValue = false;
            $i += 4;
            break;
        case 'n':
            if (($i+ 3) < $str_length && substr($str, $start, 4) == "null") {
                $this->_token = self::DATUM;
            }
            $this->_tokenValue = null;
            $i += 3;
            break;
        }

        if ($this->_token != self::EOF) {
            $this->_offset = $i + 1; // Consume the last token character
            return($this->_token);
        }

        $chr = $str{$i};
        if ($chr == '-' || $chr == '.' || ($chr >= '0' && $chr <= '9')) {
            if (preg_match(
                '/-?([0-9])*(\.[0-9]*)?((e|E)((-|\+)?)[0-9]+)?/s',
                $str, $matches, PREG_OFFSET_CAPTURE, $start
            ) && $matches[0][1] == $start
            ) {

                $datum = $matches[0][0];

                if (is_numeric($datum)) {
                    if (preg_match('/^0\d+$/', $datum)) {
                        throw new Zend_Json_Exception("Octal notation not supported by JSON (value: $datum)");
                    } else {
                        $val  = intval($datum);
                        $fVal = floatval($datum);
                        $this->_tokenValue = ($val == $fVal ? $val : $fVal);
                    }
                } else {
                    throw new Zend_Json_Exception("Illegal number format: $datum");
                }

                $this->_token = self::DATUM;
                $this->_offset = $start + strlen($datum);
            }
        } else {
            throw new Zend_Json_Exception('Illegal Token');
        }

        return($this->_token);
    }
590878
public String nextToken() {
        /*
         * If next position already computed in hasMoreElements() and
         * delimiters have changed between the computation and this invocation,
         * then use the computed value.
         */

        currentPosition = (newPosition >= 0 && !delimsChanged) ?
            newPosition : skipDelimiters(currentPosition);

        /* Reset these anyway */
        delimsChanged = false;
        newPosition = -1;

        if (currentPosition >= maxPosition)
            throw new NoSuchElementException();
        int start = currentPosition;
        currentPosition = scanToken(currentPosition);
        return str.substring(start, currentPosition);
    }
724002
def next_token(self, tokenum, value, scol):
        """"""

        # Make self.current reflect these values
        self.current.set(tokenum, value, scol)

        # Determine indent_type based on this token
        if self.current.tokenum == INDENT and self.current.value:
            self.indent_type = self.current.value[0]

        # Only proceed if we shouldn't ignore this token
        if not self.ignore_token():
            # Determining if this token is whitespace
            self.determine_if_whitespace()

            # Determine if inside a container
            self.determine_inside_container()

            # Change indentation as necessary
            self.determine_indentation()

            # See if we are force inserting this token
            if self.forced_insert():
                return

            # If we have a newline after an inserted line, then we don't need to worry about semicolons
            if self.inserted_line and self.current.tokenum == NEWLINE:
                self.inserted_line = False

            # If we have a non space, non comment after an inserted line, then insert a semicolon
            if self.result and not self.is_space and self.inserted_line:
                if self.current.tokenum != COMMENT:
                    self.result.append((OP, ';'))
                self.inserted_line = False

            # Progress the tracker
            self.progress()

            # Add a newline if we just skipped a single
            if self.single and self.single.skipped:
                self.single.skipped = False
                self.result.append((NEWLINE, '\n'))

            # Set after_space so next line knows if it is after space
            self.after_space = self.is_space
456408
public String getNextString()
    {
        Object data = this.getNextData();
        if (data instanceof String)
            return (String)data;
        return null;    // EOF
    }
650801
protected function next()
    {
        while ($token = array_shift($this->tokens)) {
            $this->line += substr_count($this->value($token), "\n");

            if (is_array($token) && in_array($token[0], [T_WHITESPACE, T_COMMENT, T_DOC_COMMENT], true)) {
                continue;
            }

            return $token;
        }
    }