assumes default token comparator
188167
public static MatchTable create(VariantGraph graph, Iterable<Token> witness) {
        Comparator<Token> comparator = new EqualityTokenComparator();
        return MatchTableImpl.create(graph, witness, comparator);
    }
771013
public final void nextToken() {
        skipIgnoredToken();
        if (isVariableBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanVariable();
        } else if (isNCharBegin()) {
            currentToken = new Tokenizer(input, dictionary, ++offset).scanChars();
        } else if (isIdentifierBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanIdentifier();
        } else if (isHexDecimalBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanHexDecimal();
        } else if (isNumberBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanNumber();
        } else if (isSymbolBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanSymbol();
        } else if (isCharsBegin()) {
            currentToken = new Tokenizer(input, dictionary, offset).scanChars();
        } else if (isEnd()) {
            currentToken = new Token(Assist.END, "", offset);
        } else {
            throw new SQLParsingException(this, Assist.ERROR);
        }
        offset = currentToken.getEndPosition();
    }
1354047
def get_ordering(self):
        """"""
        _, ordering = self.token_next_by(t=T.Keyword.Order)
        return ordering.normalized if ordering else None
793335
function token(value) {
		if ( !arguments.length ) return cred.token;
		cred.token = (value && value.length > 0 ? value : undefined);
		return that;
	}
196667
private static String initialise(Token currentToken,
                           int[][] expectedTokenSequences,
                           String[] tokenImage) {
    StringBuffer expected = new StringBuffer();
    
    //149852 - This was removed due to a findBugs error we felt was worthy of a change.
    //Added in a doPrivileged call around the getProperty call
    String eol;
    if(System.getSecurityManager() != null){
    	eol = AccessController.doPrivileged(
                new PrivilegedAction<String>(){
                    @Override
                    public String run() {
                        return System.getProperty("line.separator", "\n");
                    }
                }
  		  );
    } else {
    	eol = System.getProperty("line.separator", "\n");
    }
    int maxSize = 0;
    for (int i = 0; i < expectedTokenSequences.length; i++) {
      if (maxSize < expectedTokenSequences[i].length) {
        maxSize = expectedTokenSequences[i].length;
      }
      for (int j = 0; j < expectedTokenSequences[i].length; j++) {
        expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' ');
      }
      if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
        expected.append("...");
      }
      expected.append(eol).append("    ");
    }
    String retval = "Encountered \"";
    Token tok = currentToken.next;
    for (int i = 0; i < maxSize; i++) {
      if (i != 0) retval += " ";
      if (tok.kind == 0) {
        retval += tokenImage[0];
        break;
      }
      retval += " " + tokenImage[tok.kind];
      retval += " \"";
      retval += add_escapes(tok.image);
      retval += " \"";
      tok = tok.next;
    }
    retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn;
    retval += "." + eol;
    if (expectedTokenSequences.length == 1) {
      retval += "Was expecting:" + eol + "    ";
    } else {
      retval += "Was expecting one of:" + eol + "    ";
    }
    retval += expected.toString();
    return retval;
  }
725344
function getExpectedIndent (tokens) {
    const trivial = isTrivialToken(tokens[0])
    let expectedIndent = Number.MAX_SAFE_INTEGER

    for (let i = 0; i < tokens.length; ++i) {
      const token = tokens[i]
      const offsetInfo = offsets.get(token)

      // If the first token is not trivial then ignore trivial following tokens.
      if (offsetInfo != null && (trivial || !isTrivialToken(token))) {
        if (offsetInfo.expectedIndent != null) {
          expectedIndent = Math.min(expectedIndent, offsetInfo.expectedIndent)
        } else {
          const baseOffsetInfo = offsets.get(offsetInfo.baseToken)
          if (baseOffsetInfo != null && baseOffsetInfo.expectedIndent != null && (i === 0 || !baseOffsetInfo.baseline)) {
            expectedIndent = Math.min(expectedIndent, baseOffsetInfo.expectedIndent + offsetInfo.offset * options.indentSize)
            if (baseOffsetInfo.baseline) {
              break
            }
          }
        }
      }
    }

    return expectedIndent
  }
458483
function defaultCompareKeysFunction (a, b) {
  if (a < b) { return -1; }
  if (a > b) { return 1; }
  if (a === b) { return 0; }

  var err = new Error("Couldn't compare elements");
  err.a = a;
  err.b = b;
  throw err;
}
825768
private TextToken text(Token prev) {
    int startIndex = curCharIndex;

    while (curChar != '{' && curChar != EOF) {
      consume();
    }
    return new TextToken(prev, curCharIndex - startIndex);
  }
1130319
function defaultTokenizer(chars, wordOptions) {
        var word = chars.join(""), result, tokenRanges = [];

        function createTokenRange(start, end, isWord) {
            tokenRanges.push( { start: start, end: end, isWord: isWord } );
        }

        // Match words and mark characters
        var lastWordEnd = 0, wordStart, wordEnd;
        while ( (result = wordOptions.wordRegex.exec(word)) ) {
            wordStart = result.index;
            wordEnd = wordStart + result[0].length;

            // Create token for non-word characters preceding this word
            if (wordStart > lastWordEnd) {
                createTokenRange(lastWordEnd, wordStart, false);
            }

            // Get trailing space characters for word
            if (wordOptions.includeTrailingSpace) {
                while ( nonLineBreakWhiteSpaceRegex.test(chars[wordEnd]) ) {
                    ++wordEnd;
                }
            }
            createTokenRange(wordStart, wordEnd, true);
            lastWordEnd = wordEnd;
        }

        // Create token for trailing non-word characters, if any exist
        if (lastWordEnd < chars.length) {
            createTokenRange(lastWordEnd, chars.length, false);
        }

        return tokenRanges;
    }
271891
function (value) {
        if (value === null || value === undefined || value === defaultEqualityComparer) {
            return defaultEqualityComparer;
        }

        else if (value instanceof EqualityComparer) {
            return value;
        }

        else if (isFunction(value.hash) && isFunction(value.equals)) {
            return new EqualityComparer(value.hash, value.equals);
        }

        return defaultEqualityComparer;
    }
415931
function overrideLunrTokenizer() {

		var origTokenizer = lunr.tokenizer;
		var rSeparators = /[-./#_,;\(\)=><|]/g;

		lunr.tokenizer = function(str) {
			return origTokenizer.call(lunr, str).reduce( function (result, token) {
				if ( rSeparators.test(token) ) {
					token = token.replace(rSeparators, " ");
					result.push.apply(result, token.toLowerCase().split(/ +/));
				} else {
					result.push(token.toLowerCase());
				}
				return result;
			}, []);
		};
		Object.keys(origTokenizer).forEach(function (key) {
			lunr.tokenizer[key] = origTokenizer[key];
		});
	}
358454
public Comparator<Integer> getCompByName() {
    return new Comparator<Integer>() {
      @Override
      public int compare(Integer t1, Integer t2) {
        Integer p1 = tokenCollection.get(t1).getPositionStart();
        Integer p2 = tokenCollection.get(t2).getPositionStart();
        assert p1 != null : "no position for " + tokenCollection.get(t1);
        assert p2 != null : "no position for " + tokenCollection.get(t2);
        if (p1.equals(p2)) {
          Integer o1 = tokenCollection.get(t1).getOffsetStart();
          Integer o2 = tokenCollection.get(t2).getOffsetStart();
          if (o1 != null && o2 != null) {
            if (o1.equals(o2)) {
              return tokenCollection.get(t1).getValue()
                  .compareTo(tokenCollection.get(t2).getValue());
            } else {
              return o1.compareTo(o2);
            }
          } else {
            return tokenCollection.get(t1).getValue()
                .compareTo(tokenCollection.get(t2).getValue());
          }
        }
        return p1.compareTo(p2);
      }
    };
  }
1272337
function (token) {
        token.consume("(");
        var ast = this.parse_spec(token);
        while (token.peek() === "|") {
            token.consume("|");
            var child = this.parse_spec(token); /*  RECURSION  */
            ast = { type: "or", op1: ast, op2: child };
        }
        token.consume(")");
        return ast;
    }
888462
def token(cls: Type[ConditionType], left: Any, op: Optional[Any] = None,
              right: Optional[Any] = None) -> ConditionType:
        
        condition = cls()
        condition.left = left
        if op:
            condition.op = op
        if right:
            condition.right = right
        return condition
1028036
public static String getProtocolByToken(String token, boolean returnDefault) {
        if (StringUtils.isEmpty(token) && returnDefault) {
            return JaxWsConstants.SOAP11HTTP_BINDING;
        }

        if (JaxWsConstants.SOAP11_HTTP_TOKEN.equals(token)) {
            return JaxWsConstants.SOAP11HTTP_BINDING;
        } else if (JaxWsConstants.SOAP11_HTTP_MTOM_TOKEN.equals(token)) {
            return JaxWsConstants.SOAP11HTTP_MTOM_BINDING;
        } else if (JaxWsConstants.SOAP12_HTTP_TOKEN.equals(token)) {
            return JaxWsConstants.SOAP12HTTP_BINDING;
        } else if (JaxWsConstants.SOAP12_HTTP_MTOM_TOKEN.equals(token)) {
            return JaxWsConstants.SOAP12HTTP_MTOM_BINDING;
        } else if (JaxWsConstants.XML_HTTP_TOKEN.equals(token)) {
            return JaxWsConstants.HTTP_BINDING;
        } else {
            return token;
        }
    }
394507
private static String initialise(Token currentToken,
                           int[][] expectedTokenSequences,
                           String[] tokenImage) {
    String eol = System.getProperty("line.separator", "\n");
    StringBuffer expected = new StringBuffer();
    int maxSize = 0;
    for (int i = 0; i < expectedTokenSequences.length; i++) {
      if (maxSize < expectedTokenSequences[i].length) {
        maxSize = expectedTokenSequences[i].length;
      }
      for (int j = 0; j < expectedTokenSequences[i].length; j++) {
        expected.append(tokenImage[expectedTokenSequences[i][j]]).append(' ');
      }
      if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
        expected.append("...");
      }
      expected.append(eol).append("    ");
    }
    String retval = "Encountered \"";
    Token tok = currentToken.next;
    for (int i = 0; i < maxSize; i++) {
      if (i != 0) retval += " ";
      if (tok.kind == 0) {
        retval += tokenImage[0];
        break;
      }
      retval += " " + tokenImage[tok.kind];
      retval += " \"";
      retval += add_escapes(tok.image);
      retval += " \"";
      tok = tok.next;
    }
    retval += "\" at line " + currentToken.next.beginLine + ", column " + currentToken.next.beginColumn;
    retval += "." + eol;
    if (expectedTokenSequences.length == 1) {
      retval += "Was expecting:" + eol + "    ";
    } else {
      retval += "Was expecting one of:" + eol + "    ";
    }
    retval += expected.toString();
    return retval;
  }
1041617
public int precedence(ParserToken token) {
      if (isInfix(token)) {
         return infixHandlers.get(token.type).precedence();
      }
      return 0;
   }