lex function that supports token stacks
189904
function tokenStackLex() {
    var token;
    token = tstack.pop() || lexer.lex() || EOF;
    // if token isn't its numeric value, convert
    if (typeof token !== 'number') {
        if (token instanceof Array) {
            tstack = token;
            token = tstack.pop();
        }
        token = self.symbols_[token] || token;
    }
    return token;
}
913475
function buildPredicateFromToken(token, root) {
  return node => token.body.every(bodyToken => nodeMatchesToken(node, bodyToken, root));
}
637425
function Lexer(str, options) {
  options = options || {};
  this.stash = [];
  this.indentStack = [];
  this.indentRe = null;
  this.lineno = 1;
  this.column = 1;

  // HACK!
  function comment(str, val, offset, s) {
    var inComment = s.lastIndexOf('/*', offset) > s.lastIndexOf('*/', offset)
      , commentIdx = s.lastIndexOf('//', offset)
      , i = s.lastIndexOf('\n', offset)
      , double = 0
      , single = 0;

    if (~commentIdx && commentIdx > i) {
      while (i != offset) {
        if ("'" == s[i]) single ? single-- : single++;
        if ('"' == s[i]) double ? double-- : double++;

        if ('/' == s[i] && '/' == s[i + 1]) {
          inComment = !single && !double;
          break;
        }
        ++i;
      }
    }

    return inComment
      ? str
      : ((val === ',' && /^[,\t\n]+$/.test(str)) ? str.replace(/\n/, '\r') : val + '\r');
  };

  // Remove UTF-8 BOM.
  if ('\uFEFF' == str.charAt(0)) str = str.slice(1);

  this.str = str
    .replace(/\s+$/, '\n')
    .replace(/\r\n?/g, '\n')
    .replace(/\\ *\n/g, '\r')
    .replace(/([,(:](?!\/\/[^ ])) *(?:\/\/[^\n]*|\/\*.*?\*\/)?\n\s*/g, comment)
    .replace(/\s*\n[ \t]*([,)])/g, comment);
}
886761
function(context: ParserContext): ?Token {
    const delimiter = context.reader.peek();
    if (delimiter !== '"' && delimiter !== "'") return null;
    if (!isInsideOpenBracket(context)) return null;

    // read until the delimiter
    let stringValue = context.reader.read();
    while (!context.reader.isEOF()) {
      const next = context.reader.read();
      stringValue += next;

      if (next === delimiter) break;
    }

    return context.createToken("string", stringValue);
  }
676880
function(lexer, text) {
    // operators e.g. "--" need to return from e.g. "--x" even
    // though the "-" has been defined as a non-terminating
    // character (maybe this can be simplified!!??)
    var token = lexer.next_token(text, {matchPartial: true});
    return sl.atom(text, {token: token});
  }
162706
function parse(tokens, basePath, iterSuffix) {
  const len = tokens.length;
  const body = [];

  let i = 0;
  while (i < len) {
    const token = tokens[i];
    if (subroutines[token.tokenType]) {
      const rest = tokens.slice(i + 1);
      const [
        branches,
        tokensHandled,
        shouldBreak,
      ] = subroutines[token.tokenType](token, rest, basePath, iterSuffix);

      body.push(...branches);

      i += tokensHandled;

      if (shouldBreak) {
        break;
      }
    } else {
      i += 1;
    }
  }

  return [body, i + 1];
}
459253
function parseStatements(lex, endTokens, isRoot) {
    endTokens = Array.isArray(endTokens) ? endTokens : [endTokens];
    var statements = [];
    var temp = lex.peek();
    while (endTokens.indexOf(temp) === -1 &&
           (temp.type && endTokens.indexOf(temp.type) === -1)) {
        var statement = parseStatement(lex, isRoot);
        if (!statement) {
            throw new Error('Invalid statement');
        }
        temp = lex.peek();
        statements.push(statement);
    }
    return statements;
}
84249
function process_token(token, index, tokens) {
	var last = (index === (tokens.length - 1));
	if (token[0] === "+") return process_single(token, last);
	else if (token[0] === "#") return process_multi(token, last);
	else return process_raw(token, last);
}
1082567
def _scan_for_tokens(contents):
    """"""
    # Regexes are in priority order. Changing the order may alter the
    # behavior of the lexer
    scanner = re.Scanner([
        # Things inside quotes
        (r"(?<![^\s\(])([\"\'])(?:(?=(\\?))\2.)*?\1(?![^\s\)])",
         lambda s, t: (TokenType.QuotedLiteral, t)),
        # Numbers on their own
        (r"(?<![^\s\(])-?[0-9]+(?![^\s\)\(])", lambda s, t: (TokenType.Number,
                                                             t)),
        # Left Paren
        (r"\(", lambda s, t: (TokenType.LeftParen, t)),
        # Right Paren
        (r"\)", lambda s, t: (TokenType.RightParen, t)),
        # Either a valid function name or variable name.
        (r"(?<![^\s\(])[a-zA-z_][a-zA-Z0-9_]*(?![^\s\)\(])",
         lambda s, t: (TokenType.Word, t)),
        # Variable dereference.
        (r"(?<![^\s\(])\${[a-zA-z_][a-zA-Z0-9_]*}(?![^\s\)])",
         lambda s, t: (TokenType.Deref, t)),
        # Newline
        (r"\n", lambda s, t: (TokenType.Newline, t)),
        # Whitespace
        (r"\s+", lambda s, t: (TokenType.Whitespace, t)),
        # The beginning of a double-quoted string, terminating at end of line
        (r"(?<![^\s\(\\])[\"]([^\"]|\\[\"])*$",
         lambda s, t: (TokenType.BeginDoubleQuotedLiteral, t)),
        # The end of a double-quoted string
        (r"[^\s]*(?<!\\)[\"](?![^\s\)])",
         lambda s, t: (TokenType.EndDoubleQuotedLiteral, t)),
        # The beginning of a single-quoted string, terminating at end of line
        (r"(?<![^\s\(\\])[\']([^\']|\\[\'])*$",
         lambda s, t: (TokenType.BeginSingleQuotedLiteral, t)),
        # The end of a single-quoted string
        (r"[^\s]*(?<!\\)[\'](?![^\s\)])",
         lambda s, t: (TokenType.EndSingleQuotedLiteral, t)),
        # Begin-RST Comment Block
        (r"#.rst:$", lambda s, t: (TokenType.BeginRSTComment, t)),
        # Begin Inline RST
        (r"#\[=*\[.rst:$", lambda s, t: (TokenType.BeginInlineRST, t)),
        # End Inline RST
        (r"#\]=*\]$", lambda s, t: (TokenType.EndInlineRST, t)),
        # Comment
        (r"#", lambda s, t: (TokenType.Comment, t)),
        # Catch-all for literals which are compound statements.
        (r"([^\s\(\)]+|[^\s\(]*[^\)]|[^\(][^\s\)]*)",
         lambda s, t: (TokenType.UnquotedLiteral, t))
    ])

    tokens_return = []

    lines = contents.splitlines(True)
    lineno = 0
    for line in lines:
        lineno += 1
        col = 1

        tokens, remaining = scanner.scan(line)
        if remaining != "":
            msg = "Unknown tokens found on line {0}: {1}".format(lineno,
                                                                 remaining)
            raise RuntimeError(msg)

        for token_type, token_contents in tokens:
            tokens_return.append(Token(type=token_type,
                                       content=token_contents,
                                       line=lineno,
                                       col=col))

            col += len(token_contents)

    return tokens_return
348574
def lex(self, text):
        
        pos = 0
        while text:
            token_class, match = self.tokens.get_token(text)
            if token_class is not None:
                matched_text = text[match.start():match.end()]
                yield token_class(matched_text)
                text = text[match.end():]
                pos += match.end()
            elif text[0] in self.blank_chars:
                text = text[1:]
                pos += 1
            else:
                raise LexerError(
                        'Invalid character %s in %s' % (text[0], text),
                        position=pos)

        yield self.end_token()
583992
def evaluate_postfix(tokens):
    
    stack = []

    for token in tokens:
        total = None

        if is_int(token) or is_float(token) or is_constant(token):
            stack.append(token)
        elif is_unary(token):
            a = stack.pop()
            total = mathwords.UNARY_FUNCTIONS[token](a)
        elif len(stack):
            b = stack.pop()
            a = stack.pop()
            if token == '+':
                total = a + b
            elif token == '-':
                total = a - b
            elif token == '*':
                total = a * b
            elif token == '^':
                total = a ** b
            elif token == '/':
                if Decimal(str(b)) == 0:
                    total = 'undefined'
                else:
                    total = Decimal(str(a)) / Decimal(str(b))
            else:
                raise PostfixTokenEvaluationException(
                    'Unknown token {}'.format(token)
                )

        if total is not None:
            stack.append(total)

    # If the stack is empty the tokens could not be evaluated
    if not stack:
        raise PostfixTokenEvaluationException(
            'The postfix expression resulted in an empty stack'
        )

    return stack.pop()
996075
function bindSubtituteFnsWithTokens(actions, tokens) {
  return actions.map(function (action) {
    return Object.keys(action).reduce(function (accum, key) {
      // if it is list of sub keys, and is a fn, then bind with tokens
      if (SUBSTITUTED_KEYS.indexOf(key) !== -1 && typeof action[key] === 'function') {
        accum[key] = action[key].bind(null, tokens);
      } else {
        accum[key] = action[key];
      }
      return accum;
    }, {});
  });
}
847173
def lexical(token):
    
    lowercase = token.lower()
    first4 = lowercase[:4]
    last4 = lowercase[-4:]
    return OrderedDict([
            ('lowercase', lowercase), 
            ('first4', first4),
            ('last4', last4)
            ])
254061
function parseArgumentList() {
                    var token, expr, args = [];

                    while (true) {
                        expr = parseExpression();
                        if (typeof expr === 'undefined') {
                            break;
                        }
                        args.push(expr);
                        token = lexer.peek();
                        if (!matchOp(token, ',')) {
                            break;
                        }
                        lexer.next();
                    }

                    return args;
                }
448999
def parse_arguments(lexer: Lexer, is_const: bool) -> List[ArgumentNode]:
    """"""
    item = parse_const_argument if is_const else parse_argument
    return (
        cast(
            List[ArgumentNode],
            many_nodes(lexer, TokenKind.PAREN_L, item, TokenKind.PAREN_R),
        )
        if peek(lexer, TokenKind.PAREN_L)
        else []
    )
1214031
function symbol(lexer, text) {
  var token = lexer.next_token(text);
  return sl.atom(text, {token: token});
}
35232
def _indices(lexer):
    """"""
    first = _expect_token(lexer, {IntegerToken}).value
    rest = _zom_index(lexer)
    return (first, ) + rest