Get the i'th token, and if i is one past the end, then scan
        for another token; restrict is a list of tokens that
        are allowed, or 0 for any token.
420334
def token(self, i, restrict=None):
        
        tokens_len = len(self.tokens)
        if i == tokens_len:  # We are at the end, ge the next...
            tokens_len += self.scan(restrict)
        if i < tokens_len:
            if restrict and self.restrictions[i] and restrict > self.restrictions[i]:
                raise NotImplementedError(
                    "Unimplemented: restriction set changed")
            return self.tokens[i]
        raise NoMoreTokens()
1063417
function checkBlockdecl(i) {
  let l;

  if (i >= tokensLength) return 0;

  if (l = checkBlockdecl7(i)) tokens[i].bd_type = 7;
  else if (l = checkBlockdecl5(i)) tokens[i].bd_type = 5;
  else if (l = checkBlockdecl6(i)) tokens[i].bd_type = 6;
  else if (l = checkBlockdecl1(i)) tokens[i].bd_type = 1;
  else if (l = checkBlockdecl2(i)) tokens[i].bd_type = 2;
  else if (l = checkBlockdecl3(i)) tokens[i].bd_type = 3;
  else if (l = checkBlockdecl4(i)) tokens[i].bd_type = 4;
  else return 0;

  return l;
}
1221917
function checkUri(i) {
  const start = i;

  if (i >= tokensLength || tokens[i++].value !== 'url' ||
      i >= tokensLength || tokens[i].type !== TokenType.LeftParenthesis)
    return 0;

  return tokens[i].right - start + 1;
}
1019428
def restrict_to(self, restriction):
        
        if type(restriction) != list:
            restriction = [ restriction ]
        self._restriction = restriction
558382
def _find_end_token(self, end_token, operation):
        
        ibuffer, icache = operation.state
        length = operation.buffered[ibuffer][2]
        result = None

        for i in range(len(operation.buffered) - ibuffer - 1):
            linenum, statement, charlength = operation.buffered[i + ibuffer + 1]
            length += charlength
            
            if end_token in statement.lower():
                #We already have the absolute char index for the start of the
                #instance; we just need to update the end.
                result = (i + ibuffer + 1, length)
                break        

        #If we didn't find a terminating statement, the full length of the operation
        #is a good estimate for the extent of the instance.
        if result is None:
            result = (None, length)

        return result
1053497
def _parse_next_token(self):
        """"""
        while self._position < self.limit:
            token = self._next_pattern()
            if token:
                return token

        return None
1274642
function checkInclude(i) {
  let l;

  if (i >= tokensLength) return 0;

  if (l = checkInclude1(i)) tokens[i].include_type = 1;
  else if (l = checkInclude2(i)) tokens[i].include_type = 2;

  return l;
}
933335
function (startToken, endToken, types) {
            var is = true;
            var token = startToken;
            while (token.next && token.next !== endToken) {
                token = token.next;
                if (types.indexOf(token.type) === -1) {
                    is = false;
                    break;
                }
            }
            return is;
        }
208253
function checkBlock(i) {
  return i < tokensLength && tokens[i].type === TokenType.LeftCurlyBracket ?
      tokens[i].right - i + 1 : 0;
}
352262
def token_scan(cls, result_key, token):
        
        def _scan(self):
            return token in self

        cls.scan(result_key, _scan)
831319
def parse(self):
        """"""
        expr = self.expression()
        if not isinstance(self.current_token, EndToken):
            raise InvalidTokenError("Unconsumed trailing tokens.")
        return expr
927406
function checkUri1(i) {
  const start = i;
  let l;

  if (l = checkSC(i)) i += l;

  while (i < tokensLength) {
    if (l = checkInterpolation(i) || checkUriRaw(i)) i += l;
    else break;
  }

  if (l = checkSC(i)) i += l;

  // Check that we are at the end of the uri
  if (i < tokens[start - 1].right) return 0;

  tokens[start].uri_end = i;

  return i - start;
}
86588
private boolean tokenTextsDoNotMatch(Token token1, Token token2) {
        return !token1.getText().equals(token2.getText());
    }
454068
private Token lookahead(int n) {
        Token current = token;
        for (int i = 0; i < n; i++) {
            if (current.next == null) {
                current.next = token_source.getNextToken();
            }
            current = current.next;
        }
        return current;
    }
1346470
private void checkForTokens( String content, int startOffset, int endOffset ) {
        while( startOffset <= endOffset ) {
            // skip the delimiters to find the start of a new token
            while( isDelimiter(content.substring(startOffset, startOffset + 1)) ) {
                if (startOffset < endOffset)
                    startOffset++;
                else
                    return;
            }
            // Extract and process the entire token
            if (isQuoteDelimiter(content.substring(startOffset, startOffset + 1)))
                startOffset = getQuoteToken(content, startOffset, endOffset);
            else
                startOffset = getOtherToken(content, startOffset, endOffset);
        }
    }
285339
private Token scanText(int c) throws IOException {
        // Read first character in text so that source info does not include
        // tags.
        c = mSource.read();

        int startLine = mSource.getLineNumber();
        int startPos = mSource.getStartPosition();
        int endPos = mSource.getEndPosition();
        StringBuilder buf = new StringBuilder(256);

        while (c != -1) {
            if (c == SourceReader.ENTER_CODE) {
                if (mEmitSpecial) {
                    mLookahead.push(makeStringToken(Token.ENTER_CODE,
                                                    mSource.getBeginTag()));
                }
                break;
            }
            else if (c == SourceReader.ENTER_TEXT) {
                buf.append(mSource.getEndTag());
            }
            else {
                buf.append((char)c);
            }

            if (mSource.peek() < 0) {
                endPos = mSource.getEndPosition();
            }

            c = mSource.read();
        }

        if (c == -1) {
            // If the last token in the source file is text, trim all trailing
            // whitespace from it.

            int length = buf.length();
            
            int i;
            for (i = length - 1; i >= 0; i--) {
                if (buf.charAt(i) > ' ') {
                    break;
                }
            }

            buf.setLength(i + 1);
        }

        String str = buf.toString();
        return new StringToken(startLine, startPos, endPos,
                               Token.STRING, str);
    }
1112794
private void scanOperator() {
        while (true) {
            reader.putChar(false);
            Name newname = reader.name();
            TokenKind tk1 = tokens.lookupKind(newname);
            if (tk1 == TokenKind.IDENTIFIER) {
                reader.sp--;
                break;
            }
            tk = tk1;
            reader.scanChar();
            if (!isSpecial(reader.ch)) break;
        }
    }