Scans the first character of a token and decides, what type it is.
480542
private void scanStart() {

        if (isNumber(mInput)) {
            mState = State.NUMBER;
            mOutput.append(mInput);
            mType = TokenType.VALUE; // number
        } else if (isFirstLetter(mInput)) {
            mState = State.TEXT; // word
            mOutput.append(mInput);
            mType = TokenType.TEXT;
        } else if (isSpecial(mInput)) {
            mState = State.SPECIAL; // special character with only one digit
            mOutput.append(mInput);
            mType = retrieveType(mInput);
            mFinnished = true;
        } else if (isTwoDigistSpecial(mInput)) {
            mState = State.SPECIAL2; // 2 digit special character
            mOutput.append(mInput);
            mType = retrieveType(mInput);
        } else if ((mInput == ' ') || (mInput == '\n')) {
            mState = State.START;
            mOutput.append(mInput);
            mFinnished = true;
            mType = TokenType.SPACE;
        } else if (mInput == '#') {
            mType = TokenType.END; // end of query
            mFinnished = true;
            mPos--;
        } else {
            mState = State.UNKNOWN; // unknown character
            mOutput.append(mInput);
            mFinnished = true;
        }
        mPos++;
    }
1131463
function checkTypeSelector(i) {
  if (i >= tokensLength) return 0;

  const start = i;
  let l;

  if (l = checkNamePrefix(i)) i += l;

  if (l = checkIdentOrInterpolation(i)) i += l;
  else return 0;

  return i - start;
}
1308672
def scan(self, restrict):
        
        # Keep looking for a token, ignoring any in self.ignore
        while True:
            # Search the patterns for a match, with earlier
            # tokens in the list having preference
            best_pat = None
            best_pat_len = 0
            for p, regexp in self.patterns:
                # First check to see if we're restricting to this token
                if restrict and p not in restrict and p not in self.ignore:
                    continue
                m = regexp.match(self.input, self.pos)
                if m:
                    # We got a match
                    best_pat = p
                    best_pat_len = len(m.group(0))
                    break

            # If we didn't find anything, raise an error
            if best_pat is None:
                msg = "Bad Token"
                if restrict:
                    msg = "Trying to find one of " + ", ".join(restrict)
                raise SyntaxError(self.pos, msg)

            # If we found something that isn't to be ignored, return it
            if best_pat in self.ignore:
                # This token should be ignored ..
                self.pos += best_pat_len
            else:
                end_pos = self.pos + best_pat_len
                # Create a token with this data
                token = (
                    self.pos,
                    end_pos,
                    best_pat,
                    self.input[self.pos:end_pos]
                )
                self.pos = end_pos
                # Only add this token if it's not in the list
                # (to prevent looping)
                if not self.tokens or token != self.tokens[-1]:
                    self.tokens.append(token)
                    self.restrictions.append(restrict)
                    return 1
                break
        return 0
393682
function Type1Parser_extractFontProgram() {
      var stream = this.stream;

      var subrs = [], charstrings = [];
      var program = {
        subrs: [],
        charstrings: [],
        properties: {
          'privateData': {
            'lenIV': 4
          }
        }
      };
      var token, length, data, lenIV, encoded;
      while ((token = this.getToken()) !== null) {
        if (token !== '/') {
          continue;
        }
        token = this.getToken();
        switch (token) {
          case 'CharStrings':
            // The number immediately following CharStrings must be greater or
            // equal to the number of CharStrings.
            this.getToken();
            this.getToken(); // read in 'dict'
            this.getToken(); // read in 'dup'
            this.getToken(); // read in 'begin'
            while(true) {
              token = this.getToken();
              if (token === null || token === 'end') {
                break;
              }

              if (token !== '/') {
                continue;
              }
              var glyph = this.getToken();
              length = this.readInt();
              this.getToken(); // read in 'RD' or '-|'
              data = stream.makeSubStream(stream.pos, length);
              lenIV = program.properties.privateData['lenIV'];
              encoded = decrypt(data.getBytes(), CHAR_STRS_ENCRYPT_KEY, lenIV);
              // Skip past the required space and binary data.
              stream.skip(length);
              this.nextChar();
              token = this.getToken(); // read in 'ND' or '|-'
              if (token === 'noaccess') {
                this.getToken(); // read in 'def'
              }
              charstrings.push({
                glyph: glyph,
                encoded: encoded
              });
            }
            break;
          case 'Subrs':
            var num = this.readInt();
            this.getToken(); // read in 'array'
            while ((token = this.getToken()) === 'dup') {
              var index = this.readInt();
              length = this.readInt();
              this.getToken(); // read in 'RD' or '-|'
              data = stream.makeSubStream(stream.pos, length);
              lenIV = program.properties.privateData['lenIV'];
              encoded = decrypt(data.getBytes(), CHAR_STRS_ENCRYPT_KEY, lenIV);
              // Skip past the required space and binary data.
              stream.skip(length);
              this.nextChar();
              token = this.getToken(); // read in 'NP' or '|'
              if (token === 'noaccess') {
                this.getToken(); // read in 'put'
              }
              subrs[index] = encoded;
            }
            break;
          case 'BlueValues':
          case 'OtherBlues':
          case 'FamilyBlues':
          case 'FamilyOtherBlues':
            var blueArray = this.readNumberArray();
            // *Blue* values may contain invalid data: disables reading of
            // those values when hinting is disabled.
            if (blueArray.length > 0 && (blueArray.length % 2) === 0 &&
                HINTING_ENABLED) {
              program.properties.privateData[token] = blueArray;
            }
            break;
          case 'StemSnapH':
          case 'StemSnapV':
            program.properties.privateData[token] = this.readNumberArray();
            break;
          case 'StdHW':
          case 'StdVW':
            program.properties.privateData[token] =
              this.readNumberArray()[0];
            break;
          case 'BlueShift':
          case 'lenIV':
          case 'BlueFuzz':
          case 'BlueScale':
          case 'LanguageGroup':
          case 'ExpansionFactor':
            program.properties.privateData[token] = this.readNumber();
            break;
          case 'ForceBold':
            program.properties.privateData[token] = this.readBoolean();
            break;
        }
      }

      for (var i = 0; i < charstrings.length; i++) {
        glyph = charstrings[i].glyph;
        encoded = charstrings[i].encoded;
        var charString = new Type1CharString();
        var error = charString.convert(encoded, subrs);
        var output = charString.output;
        if (error) {
          // It seems when FreeType encounters an error while evaluating a glyph
          // that it completely ignores the glyph so we'll mimic that behaviour
          // here and put an endchar to make the validator happy.
          output = [14];
        }
        program.charstrings.push({
          glyphName: glyph,
          charstring: output,
          width: charString.width,
          lsb: charString.lsb,
          seac: charString.seac
        });
      }

      return program;
    }
535767
function(context: ParserContext): ?Token {
    if (!context.reader.match("/")) return null;

    const previousNonWsToken = context.token(context.count() - 1);
    let previousToken = null;
    if (context.defaultData.text !== "")
      previousToken = context.createToken("default", context.defaultData.text);

    if (!previousToken) previousToken = previousNonWsToken;

    if (
      // The first token of the string
      previousToken !== undefined &&
      // Since Ruby doesn't require statement terminators, if the previous token
      // was whitespace and contained a newline, then we're good.
      (previousToken.name !== "default" ||
        previousToken.value.indexOf("\n") < 0)
    ) {
      // In these case the / stands for division.
      if (
        util.contains(["keyword", "ident", "number"], previousNonWsToken.name)
      )
        return null;
      if (
        previousNonWsToken.name === "punctuation" &&
        !util.contains(["(", "{", "[", ","], previousNonWsToken.value)
      )
        return null;
    }

    // read the regex literal
    let regexLiteral = context.reader.read();
    let charClass = false;
    while (!context.reader.isEOF()) {
      const next = context.reader.read();
      regexLiteral += next;

      if (next === "\\") regexLiteral += context.reader.read();
      else if (next === "[") charClass = true;
      else if (next === "]") charClass = false;
      else if (next === "/" && !charClass) break;
    }

    // Read the regex modifiers. For the sake of simplicity we accept all
    // characters, but currently only "ioxmusen" are supported.
    while (!context.reader.isEOF() && /[A-Za-z]/.test(context.reader.peek()))
      regexLiteral += context.reader.read();

    return context.createToken("regexLiteral", regexLiteral);
  }
1365602
private Token scanNumber(int c) throws IOException {
        int startLine = mSource.getLineNumber();
        int startPos = mSource.getStartPosition();
        mWord.setLength(0);
        
        int errorPos = -1;

        // 0 is decimal int, 
        // 1 is hex int, 
        // 2 is decimal long, 
        // 3 is hex long, 
        // 4 is float, 
        // 5 is double,
        // 6 is auto-double by decimal
        // 7 is auto-double by exponent ('e' or 'E')
        int type = 0; 

        if (c == '0') {
            if (mSource.peek() == 'x' || mSource.peek() == 'X') {
                type = 1;
                mSource.read(); // absorb the 'x'
                c = mSource.read(); // get the first digit after the 'x'
            }
        }

        for (; c != -1; c = mSource.read()) {
            if (c == '.') {
                int peek = mSource.peek();
                if (peek == '.') {
                    mSource.unread();
                    break;
                }
                else {
                    if (peek < '0' || peek > '9') {
                        error("number.decimal.end");
                    }

                    mWord.append((char)c);

                    if (type == 0) {
                        type = 6;
                    }
                    else if (errorPos < 0) {
                        errorPos = mSource.getStartPosition();
                    }

                    continue;
                }
            }

            if (c >= '0' && c <= '9') {
                mWord.append((char)c);

                if (type == 2 || type == 3 || type == 4 || type == 5) {
                    if (errorPos < 0) {
                        errorPos = mSource.getStartPosition();
                    }
                }

                continue;
            }

            if ((c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) {
                if (type == 1) {
                    mWord.append((char)c);
                    continue;
                }

                if (c == 'f' || c == 'F') {
                    if (type == 0 || type == 6 || type == 7) {
                        type = 4;
                        continue;
                    }
                }
                else if (c == 'd' || c == 'D') {
                    if (type == 0 || type == 6 || type == 7) {
                        type = 5;
                        continue;
                    }
                }
                else if (c == 'e' || c == 'E') {
                    if (type == 0 || type == 6) {
                        mWord.append((char)c);
                        type = 7;
                        int peek = mSource.peek();
                        if (peek == '+' || peek == '-') {
                            mWord.append((char)mSource.read());
                        }
                        continue;
                    }
                }

                mWord.append((char)c);

                if (errorPos < 0) {
                    errorPos = mSource.getStartPosition();
                }

                continue;
            }

            if (c == 'l' || c == 'L') {
                if (type == 0) {
                    type = 2;
                }
                else if (type == 1) {
                    type = 3;
                }
                else {
                    mWord.append((char)c);
                    if (errorPos < 0) {
                        errorPos = mSource.getStartPosition();
                    }
                }

                continue;
            }

            if (Character.isLetterOrDigit((char)c)) {
                mWord.append((char)c);

                if (errorPos < 0) {
                    errorPos = mSource.getStartPosition();
                }
            }
            else {
                mSource.unread();
                break;
            }
        }
        
        String str = mWord.toString();
        int endPos = mSource.getEndPosition();
        Token token;

        if (errorPos >= 0) {
            token = new StringToken
                (startLine, startPos, endPos, errorPos, Token.NUMBER, str);
        }
        else {
            try {
                switch (type) {
                case 0:
                default:
                    try {
                        token = new IntToken
                            (startLine, startPos, endPos,
                             Integer.parseInt(str));
                    }
                    catch (NumberFormatException e) {
                        token = new LongToken
                            (startLine, startPos, endPos, Long.parseLong(str));
                    }
                    break;
                case 1:
                    try {
                        token = new IntToken
                            (startLine, startPos, endPos, parseHexInt(str));
                    }
                    catch (NumberFormatException e) {
                        token = new LongToken
                            (startLine, startPos, endPos, parseHexLong(str));
                    }
                    break;
                case 2:
                    token = new LongToken
                        (startLine, startPos, endPos, Long.parseLong(str));
                    break;
                case 3:
                    token = new LongToken
                        (startLine, startPos, endPos, parseHexLong(str));
                    break;
                case 4:
                    token = new FloatToken
                        (startLine, startPos, endPos, Float.parseFloat(str));
                    break;
                case 5:
                case 6:
                case 7:
                    token = new DoubleToken
                        (startLine, startPos, endPos, Double.parseDouble(str));
                    break;
                }
            }
            catch (NumberFormatException e) {
                token = new IntToken(startLine, startPos, endPos, 0);
                error("number.range", token.getSourceInfo());
            }
        }

        return token;
    }
248307
function scanHexLiteral(start) {
  var number = '';

  while (index < length) {
    if (!isHexDigit(source[index])) {
      break;
    }
    number += source[index++];
  }

  if (number.length === 0) {
    throwError({}, MessageUnexpectedToken, ILLEGAL);
  }

  if (isIdentifierStart(source.charCodeAt(index))) {
    throwError({}, MessageUnexpectedToken, ILLEGAL);
  }

  return {
    type: TokenNumericLiteral,
    value: parseInt('0x' + number, 16),
    start: start,
    end: index
  };
}
816058
Type readType(int i) {
        int index = poolIdx[i];
        return sigToType(buf, index + 3, getChar(index + 1));
    }
1112794
private void scanOperator() {
        while (true) {
            reader.putChar(false);
            Name newname = reader.name();
            TokenKind tk1 = tokens.lookupKind(newname);
            if (tk1 == TokenKind.IDENTIFIER) {
                reader.sp--;
                break;
            }
            tk = tk1;
            reader.scanChar();
            if (!isSpecial(reader.ch)) break;
        }
    }
935477
function checkLiteral(node) {
            const token = sourceCode.getFirstToken(node),
                nodeType = token.type,
                nodeValue = token.value;

            if (nodeType === "RegularExpression") {
                checkRegex(node, nodeValue, token.range[0]);
            }
        }
186023
def _scan(self, type):
        
        tok = self._scanner.token(self._pos, frozenset([type]))
        if tok[2] != type:
            err = SyntaxError("SyntaxError[@ char %s: %s]" % (repr(tok[0]), "Trying to find " + type))
            err.pos = tok[0]
            raise err
        self._pos += 1
        return tok[3]
2124
public void assertToken(final int token) throws ParseException {
    try {
      if (nextToken() != token) {
        throw new ParseException("Expected [" + token + "], read [" +
                                  ttype + "] at " + lineno());
      }

      if (debug()) {
        if (token > 0) {
          debug("[" + (char)token + "]");
        } else {
          debug("[" + token + "]");
        }
      }
    } catch (IOException e) {
      throw new ParseException(e);
    }
  }
1074309
function (checks) {
    this.from = this.char;

    // Move to the next non-space character.
    var start;
    if (/\s/.test(this.peek())) {
      start = this.char;

      while (/\s/.test(this.peek())) {
        this.from += 1;
        this.skip();
      }
    }

    // Methods that work with multi-line structures and move the
    // character pointer.

    var match = this.scanComments() ||
      this.scanStringLiteral(checks) ||
      this.scanTemplateLiteral();

    if (match) {
      return match;
    }

    // Methods that don't move the character pointer.

    match =
      this.scanRegExp() ||
      this.scanPunctuator() ||
      this.scanKeyword() ||
      this.scanIdentifier() ||
      this.scanNumericLiteral();

    if (match) {
      this.skip(match.value.length);
      return match;
    }

    // No token could be matched, give up.

    return null;
  }
178103
def get_tokens(source):
    
    if not source.endswith('\n'):
        source += '\n'

    # Cache various valid character sets for speed.
    valid_identifier_first_chars = VALID_IDENTIFIER_FIRST_CHARS
    valid_identifier_chars = VALID_IDENTIFIER_CHARS
    hex_digits = HEX_DIGITS
    int_or_float_digits = INT_OR_FLOAT_DIGITS
    int_or_float_digits2 = int_or_float_digits | set('.')

    # Ignore tokens while in a #if 0 block.
    count_ifs = 0

    i = 0
    end = len(source)
    while i < end:
        # Skip whitespace.
        while i < end and source[i].isspace():
            i += 1
        if i >= end:
            return

        token_type = UNKNOWN
        start = i
        c = source[i]
        # Find a string token.
        if c in valid_identifier_first_chars or c == '_':
            token_type = NAME
            while source[i] in valid_identifier_chars:
                i += 1
            # String and character constants can look like a name if
            # they are something like L"".
            if source[i] == "'" and source[start:i] in _STR_PREFIXES:
                token_type = CONSTANT
                i = _get_char(source, start, i)
            elif source[i] == '"' and source[start:i] in _STR_PREFIXES:
                token_type = CONSTANT
                i = _get_string(source, i)
        elif c == '/' and source[i + 1] == '/':  # Find // comments.
            i = _find(source, '\n', i)
            continue
        elif c == '/' and source[i + 1] == '*':  # Find /* comments. */
            i = _find(source, '*/', i) + 2
            continue
        elif c in '<>':                          # Handle '<' and '>' tokens.
            token_type = SYNTAX
            i += 1
            new_ch = source[i]
            # Do not merge '>>' or '>>=' into a single token
            if new_ch == c and c != '>':
                i += 1
                new_ch = source[i]
            if new_ch == '=':
                i += 1
        elif c in ':+-&|=':                      # Handle 'XX' and 'X=' tokens.
            token_type = SYNTAX
            i += 1
            new_ch = source[i]
            if new_ch == c:
                i += 1
            elif c == '-' and new_ch == '>':
                i += 1
            elif new_ch == '=':
                i += 1
        elif c in '!*^%/':                       # Handle 'X=' tokens.
            token_type = SYNTAX
            i += 1
            new_ch = source[i]
            if new_ch == '=':
                i += 1
        elif c in '()[]{}~?;.,':                 # Handle single char tokens.
            token_type = SYNTAX
            i += 1
            if c == '.' and source[i].isdigit():
                token_type = CONSTANT
                i += 1
                while source[i] in int_or_float_digits:
                    i += 1
                # Handle float suffixes.
                for suffix in ('l', 'f'):
                    if suffix == source[i:i + 1].lower():
                        i += 1
                        break
        elif c.isdigit():                        # Find integer.
            token_type = CONSTANT
            if c == '0' and source[i + 1] in 'xX':
                # Handle hex digits.
                i += 2
                while source[i] in hex_digits:
                    i += 1
            else:
                while source[i] in int_or_float_digits2:
                    i += 1
            # Handle integer (and float) suffixes.
            if source[i].isalpha():
                for suffix in ('ull', 'll', 'ul', 'l', 'f', 'u'):
                    size = len(suffix)
                    if suffix == source[i:i + size].lower():
                        i += size
                        break
        elif c == '"':                           # Find string.
            token_type = CONSTANT
            i = _get_string(source, i)
        elif c == "'":                           # Find char.
            token_type = CONSTANT
            i = _get_char(source, start, i)
        elif c == '#':                           # Find pre-processor command.
            token_type = PREPROCESSOR
            got_if = source[i:i + 3] == '#if'
            if count_ifs and source[i:i + 6] == '#endif':
                count_ifs -= 1
                if count_ifs == 0:
                    source = source[:i].ljust(i + 6) + source[i + 6:]
                    continue

            # Handle preprocessor statements (\ continuations).
            while True:
                i1 = source.find('\n', i)
                i2 = source.find('//', i)
                i3 = source.find('/*', i)
                i4 = source.find('"', i)
                # Get the first important symbol (newline, comment, EOF/end).
                i = min([x for x in (i1, i2, i3, i4, end) if x != -1])

                # Handle comments in #define macros.
                if i == i3:
                    i = _find(source, '*/', i) + 2
                    source = source[:i3].ljust(i) + source[i:]
                    continue

                # Handle #include "dir//foo.h" properly.
                if source[i] == '"':
                    i = _find(source, '"', i + 1) + 1
                    continue

                # Keep going if end of the line and the line ends with \.
                if i == i1 and source[i - 1] == '\\':
                    i += 1
                    continue

                if got_if:
                    begin = source.find('(', start, i)
                    if begin == -1:
                        begin = source.find(' ', start)
                    begin = begin + 1
                    s1 = source.find(' ', begin)
                    s2 = source.find(')', begin)
                    s3 = source.find('\n', begin)
                    s = min([x for x in (s1, s2, s3, end) if x != -1])

                    condition = source[begin:s]
                    if (
                        count_ifs or
                        condition == '0' or
                        condition == '__OBJC__'
                    ):
                        count_ifs += 1
                break
        elif c == '\\':                          # Handle \ in code.
            # This is different from the pre-processor \ handling.
            i += 1
            continue
        elif count_ifs:
            # Ignore bogus code when we are inside an #if block.
            i += 1
            continue
        else:
            raise TokenError("unexpected token '{0}'".format(c))

        if count_ifs:
            continue

        assert i > 0
        yield Token(token_type, source[start:i], start, i)
745150
private boolean isWord(String token) {
    if (token.length() == 1) {
      char c = token.charAt(0);
      if (!Character.isLetter(c)) {
        return false;
      }
    }
    return true;
  }
292089
public Token scanCharacterLiteral() {
		int start = pos;
		pos++;
		char c = input.charAt(pos++);
		if (c == '\\') {
			// escape code
			switch (input.charAt(pos++)) {
			case 'b':
				c = '\b';
				break;
			case 't':
				c = '\t';
				break;
			case 'n':
				c = '\n';
				break;
			case 'f':
				c = '\f';
				break;
			case 'r':
				c = '\r';
				break;
			case '"':
				c = '\"';
				break;
			case '\'':
				c = '\'';
				break;
			case '\\':
				c = '\\';
				break;
			default:
				syntaxError("unrecognised escape character", pos);
			}
		}
		if (input.charAt(pos) != '\'') {
			syntaxError("unexpected end-of-character", pos);
		}
		pos = pos + 1;
		return new Token(Token.Kind.CharLiteral, input.substring(start, pos),
				start);
	}
614849
function _checkUnicodeWildcard(i) {
  const start = i;

  if (i >= tokensLength) return 0;

  while (i < tokensLength) {
    if (tokens[i].type === TokenType.QuestionMark) i += 1;
    else break;
  }

  return i - start;
}