Consumes a token, first eating up any whitespace ahead of it. Note that number tokens are not necessarily valid
numbers.
430367
private Token advanceToken() throws JsonParserException {
    int c = advanceChar();
    while (isWhitespace(c))
      c = advanceChar();

    tokenLinePos = linePos;
    tokenCharPos = index - rowPos - utf8adjust;
    tokenCharOffset = charOffset + index;

    switch (c) {
    case -1:
      return token = Token.EOF;
    case '[': // Inlined function to avoid additional stack
      JsonArray list = new JsonArray();
      if (advanceToken() != Token.ARRAY_END)
        while (true) {
          list.add(currentValue());
          if (advanceToken() == Token.ARRAY_END)
            break;
          if (token != Token.COMMA)
            throw createParseException(null, "Expected a comma or end of the array instead of " + token,
                true);
          if (advanceToken() == Token.ARRAY_END)
            throw createParseException(null, "Trailing comma found in array", true);
        }
      value = list;
      return token = Token.ARRAY_START;
    case ']':
      return token = Token.ARRAY_END;
    case ',':
      return token = Token.COMMA;
    case ':':
      return token = Token.COLON;
    case '{': // Inlined function to avoid additional stack
      JsonObject map = new JsonObject();
      if (advanceToken() != Token.OBJECT_END)
        while (true) {
          if (token != Token.STRING)
            throw createParseException(null, "Expected STRING, got " + token, true);
          String key = (String)value;
          if (advanceToken() != Token.COLON)
            throw createParseException(null, "Expected COLON, got " + token, true);
          advanceToken();
          map.put(key, currentValue());
          if (advanceToken() == Token.OBJECT_END)
            break;
          if (token != Token.COMMA)
            throw createParseException(null, "Expected a comma or end of the object instead of " + token,
                true);
          if (advanceToken() == Token.OBJECT_END)
            throw createParseException(null, "Trailing object found in array", true);
        }
      value = map;
      return token = Token.OBJECT_START;
    case '}':
      return token = Token.OBJECT_END;
    case 't':
      consumeKeyword((char)c, TRUE);
      value = Boolean.TRUE;
      return token = Token.TRUE;
    case 'f':
      consumeKeyword((char)c, FALSE);
      value = Boolean.FALSE;
      return token = Token.FALSE;
    case 'n':
      consumeKeyword((char)c, NULL);
      value = null;
      return token = Token.NULL;
    case '\"':
      value = consumeTokenString();
      return token = Token.STRING;
    case '-':
    case '0':
    case '1':
    case '2':
    case '3':
    case '4':
    case '5':
    case '6':
    case '7':
    case '8':
    case '9':
      value = consumeTokenNumber((char)c);
      return token = Token.NUMBER;
    case '+':
    case '.':
      throw createParseException(null, "Numbers may not start with '" + (char)c + "'", true);
    default:
    }

    if (isAsciiLetter(c))
      throw createHelpfulException((char)c, null, 0);

    throw createParseException(null, "Unexpected character: " + (char)c, true);
  }
1144161
private KeyToken key(Token prev) {

    // Store keys as normal Strings; we don't want keys to contain spans.
    StringBuilder sb = new StringBuilder();

    // Consume the opening '{'.
    consume();
    while ((curChar >= 'a' && curChar <= 'z') || curChar == '_') {
      sb.append(curChar);
      consume();
    }

    // Consume the closing '}'.
    if (curChar != '}') {
      throw new IllegalArgumentException("Unexpected character '" + curChar
          + "'; expecting lower case a-z, '_', or '}'");
    }
    consume();

    // Disallow empty keys: {}.
    if (sb.length() == 0) {
      throw new IllegalArgumentException("Empty key: {}");
    }

    String key = sb.toString();
    keys.add(key);
    return new KeyToken(prev, key);
  }
757516
function consumeExpr(remaining) {
  var expr;
  var match;
  var name;

  for (var j = 0; j < exprs.length; j++) {
    var e = exprs[j];
    var n = e.name;
    var m = remaining.match(e.regexObj);

    if (m != null) {
      match = m;
      expr = e;
      name = n;
      var consumed = m[0];
      remaining = remaining.substring(consumed.length);
      break; // we've consumed one expr, so we can return now
    }
  }

  return {
    expr: expr,
    match: match,
    name: name,
    remaining: remaining
  };
}
69555
public void nextToken() {
        byte ct[] = ctype;
        int c;
        int lctype;
        sval = null;
        isQuoted = false;

        do {
            c = read();
            if (c < 0) {
                return;
            }
            lctype = (c < 256) ? ct[c] : unicode2ctype(c);
        } while (lctype == CT_WHITESPACE);

        if (lctype == CT_ALPHA) {
            int i = 0;
            do {
                if (i >= buf.length) {
                    buf = Arrays.copyOf(buf, buf.length * 2);
                }
                buf[i++] = (char) c;
                c = read();
                lctype = c < 0 ? CT_WHITESPACE : (c < 256)? ct[c] : unicode2ctype(c);
            } while (lctype == CT_ALPHA);
            if (c >= 0) --next; // push last back
            sval = String.copyValueOf(buf, 0, i);
            return;
        }

        if (lctype == CT_QUOTE) {
            int quote = c;
            int i = 0;
            /* Invariants (because \Octal needs a lookahead):
             *   (i)  c contains char value
             *   (ii) d contains the lookahead
             */
            int d = read();
            while (d >= 0 && d != quote) {
                if (d == '\\') {
                    c = read();
                    int first = c;   /* To allow \377, but not \477 */
                    if (c >= '0' && c <= '7') {
                        c = c - '0';
                        int c2 = read();
                        if ('0' <= c2 && c2 <= '7') {
                            c = (c << 3) + (c2 - '0');
                            c2 = read();
                            if ('0' <= c2 && c2 <= '7' && first <= '3') {
                                c = (c << 3) + (c2 - '0');
                                d = read();
                            } else
                                d = c2;
                        } else
                          d = c2;
                    } else {
                        switch (c) {
                        case 'a':
                            c = 0x7;
                            break;
                        case 'b':
                            c = '\b';
                            break;
                        case 'f':
                            c = 0xC;
                            break;
                        case 'n':
                            c = '\n';
                            break;
                        case 'r':
                            c = '\r';
                            break;
                        case 't':
                            c = '\t';
                            break;
                        case 'v':
                            c = 0xB;
                            break;
                        }
                        d = read();
                    }
                } else {
                    c = d;
                    d = read();
                }
                if (i >= buf.length) {
                    buf = Arrays.copyOf(buf, buf.length * 2);
                }
                buf[i++] = (char)c;
            }

            if (d == quote) {
                isQuoted = true;
            }
            sval = String.copyValueOf(buf, 0, i);
        }
    }
676217
private Token consumeNextToken(Token.Type... expected) {
        try {
            return tokens.consume(expected);
        } catch (UnexpectedElementException e) {
            throw new UnexpectedTokenException(e);
        }
    }
1292260
public ParserToken consume(@NonNull ParserTokenType expectedType) throws ParseException {
      ParserToken next = consume();
      if (next == null || !expectedType.isInstance(next.type)) {
         throw new ParseException("Expected the next token to be of type " + expectedType + ", but found " + (next == null
                                                                                                              ? null
                                                                                                              : next.type));
      }
      return next;
   }
1065222
def consumeNumberEntity(self, isHex):
        

        allowed = digits
        radix = 10
        if isHex:
            allowed = hexDigits
            radix = 16

        charStack = []

        # Consume all the characters that are in range while making sure we
        # don't hit an EOF.
        c = self.stream.char()
        while c in allowed and c is not EOF:
            charStack.append(c)
            c = self.stream.char()

        # Convert the set of characters consumed to an int.
        charAsInt = int("".join(charStack), radix)

        # Certain characters get replaced with others
        if charAsInt in replacementCharacters:
            char = replacementCharacters[charAsInt]
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
                                    "illegal-codepoint-for-numeric-entity",
                                    "datavars": {"charAsInt": charAsInt}})
        elif ((0xD800 <= charAsInt <= 0xDFFF) or
              (charAsInt > 0x10FFFF)):
            char = "\uFFFD"
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
                                    "illegal-codepoint-for-numeric-entity",
                                    "datavars": {"charAsInt": charAsInt}})
        else:
            # Should speed up this check somehow (e.g. move the set to a constant)
            if ((0x0001 <= charAsInt <= 0x0008) or
                (0x000E <= charAsInt <= 0x001F) or
                (0x007F <= charAsInt <= 0x009F) or
                (0xFDD0 <= charAsInt <= 0xFDEF) or
                charAsInt in frozenset([0x000B, 0xFFFE, 0xFFFF, 0x1FFFE,
                                        0x1FFFF, 0x2FFFE, 0x2FFFF, 0x3FFFE,
                                        0x3FFFF, 0x4FFFE, 0x4FFFF, 0x5FFFE,
                                        0x5FFFF, 0x6FFFE, 0x6FFFF, 0x7FFFE,
                                        0x7FFFF, 0x8FFFE, 0x8FFFF, 0x9FFFE,
                                        0x9FFFF, 0xAFFFE, 0xAFFFF, 0xBFFFE,
                                        0xBFFFF, 0xCFFFE, 0xCFFFF, 0xDFFFE,
                                        0xDFFFF, 0xEFFFE, 0xEFFFF, 0xFFFFE,
                                        0xFFFFF, 0x10FFFE, 0x10FFFF])):
                self.tokenQueue.append({"type": tokenTypes["ParseError"],
                                        "data":
                                        "illegal-codepoint-for-numeric-entity",
                                        "datavars": {"charAsInt": charAsInt}})
            try:
                # Try/except needed as UCS-2 Python builds' unichar only works
                # within the BMP.
                char = chr(charAsInt)
            except ValueError:
                v = charAsInt - 0x10000
                char = chr(0xD800 | (v >> 10)) + chr(0xDC00 | (v & 0x3FF))

        # Discard the ; if present. Otherwise, put it back on the queue and
        # invoke parseError on parser.
        if c != ";":
            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
                                    "numeric-entity-without-semicolon"})
            self.stream.unget(c)

        return char
507351
def consume_token(self, tokens, index, tokens_len):
        
        del tokens_len

        if tokens[index].type == TokenType.EndInlineRST:
            return _paste_tokens_line_by_line(tokens,
                                              TokenType.RST,
                                              self.begin,
                                              index + 1)
1174409
public void consumeExpectedSymbol(String symbol) {
        if (current().matches(Token.TokenType.SYMBOL, symbol)) {
            consume();
        } else {
            addError(current(), "Unexpected token: '%s'. Expected: '%s'", current().getSource(), symbol);
        }
    }
1157146
function integer(token, addTokenToMatch, getNextToken) {
    if (calc(token, addTokenToMatch, getNextToken)) {
        return true;
    }

    if (token === null) {
        return false;
    }

    var numberEnd = consumeNumber(token.value, false);
    if (numberEnd !== token.value.length) {
        return false;
    }

    addTokenToMatch();
    return true;
}
805008
private final boolean skip_whitespace(CommentStrategy commentStrategy) throws IOException
    {
        boolean any_whitespace = false;
        int c;

        loop: for (;;) {
            c = read_char();
            switch (c) {
            case -1:
                break loop;
            case ' ':
            case '\t':
            // new line normalization and counting is handled in read_char
            case CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_1:
            case CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_2:
            case CharacterSequence.CHAR_SEQ_NEWLINE_SEQUENCE_3:
            case CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_1:
            case CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_2:
            case CharacterSequence.CHAR_SEQ_ESCAPED_NEWLINE_SEQUENCE_3:
                any_whitespace = true;
                break;
            case '/':
                if (!commentStrategy.onComment(this))
                {
                    break loop;
                }
                any_whitespace = true;
                break;
            default:
                break loop;
            }
        }
        unread_char(c);
        return any_whitespace;
    }
918327
def consume_at_rule(input = @tokens)
      rule = {}

      rule[:tokens] = input.collect do
        rule[:name]    = input.consume[:value]
        rule[:prelude] = []

        while token = input.consume
          node = token[:node]

          if node == :comment # Non-standard.
            next

          elsif node == :semicolon
            break

          elsif node === :'{'
            # Note: The spec says the block should _be_ the consumed simple
            # block, but Simon Sapin's CSS parsing tests and tinycss2 expect
            # only the _value_ of the consumed simple block here. I assume I'm
            # interpreting the spec too literally, so I'm going with the
            # tinycss2 behavior.
            rule[:block] = consume_simple_block(input)[:value]
            break

          elsif node == :simple_block && token[:start] == '{'
            # Note: The spec says the block should _be_ the simple block, but
            # Simon Sapin's CSS parsing tests and tinycss2 expect only the
            # _value_ of the simple block here. I assume I'm interpreting the
            # spec too literally, so I'm going with the tinycss2 behavior.
            rule[:block] = token[:value]
            break

          else
            input.reconsume
            rule[:prelude] << consume_component_value(input)
          end
        end
      end

      create_node(:at_rule, rule)
    end
1320449
function (token) {
            if (token.prev.type !== 'WhiteSpace') {
                insertBefore(token, whiteSpaceFactory());
            }
            if (token.next.type !== 'WhiteSpace' && token.next.type !== 'LineBreak') {
                insertAfter(token, whiteSpaceFactory());
            }
        }
618078
private int nextToken(StreamTokenizer tokeniser, Reader in, boolean ignoreEOF) throws IOException, ParserException {
        int token = tokeniser.nextToken();
        if (!ignoreEOF && token == StreamTokenizer.TT_EOF) {
            throw new ParserException("Unexpected end of file", getLineNumber(tokeniser, in));
        }
        return token;
    }
419921
function checkSpacingBefore(token) {
            const prevToken = sourceCode.getTokenBefore(token);

            if (prevToken &&
                CLOSE_PAREN.test(token.value) &&
                astUtils.isTokenOnSameLine(prevToken, token) &&
                sourceCode.isSpaceBetweenTokens(prevToken, token) !== always
            ) {
                context.report({
                    loc: token.loc.start,
                    messageId: `${prefix}Before`,
                    fix(fixer) {
                        if (always) {
                            return fixer.insertTextBefore(token, " ");
                        }
                        return fixer.removeRange([
                            prevToken.range[1],
                            token.range[0]
                        ]);
                    }
                });
            }
        }
1049908
def ConsumeIdentifierOrNumber(self):
    
    result = self.token
    if not self._IDENTIFIER_OR_NUMBER.match(result):
      raise self.ParseError('Expected identifier or number.')
    self.NextToken()
    return result
622457
def consume_token(self, tokens, index, tokens_len):
        
        if _is_really_comment(tokens, index):
            self.last_line_with_comment = tokens[index].line

        finished = False

        if (not _is_in_comment_type(tokens[index].type) and
                self.last_line_with_comment != tokens[index].line):
            finished = True
            end = index
        elif index == (tokens_len - 1):
            finished = True
            end = index + 1

        if finished:
            return _paste_tokens_line_by_line(tokens,
                                              TokenType.RST,
                                              self.begin,
                                              end)