Constructs compiled regex to parse out reserved tokens.
486132
def _make_reserved_tokens_re(reserved_tokens):
  """"""
  if not reserved_tokens:
    return None
  escaped_tokens = [_re_escape(rt) for rt in reserved_tokens]
  pattern = "(%s)" % "|".join(escaped_tokens)
  reserved_tokens_re = _re_compile(pattern)
  return reserved_tokens_re
1082567
def _scan_for_tokens(contents):
    """"""
    # Regexes are in priority order. Changing the order may alter the
    # behavior of the lexer
    scanner = re.Scanner([
        # Things inside quotes
        (r"(?<![^\s\(])([\"\'])(?:(?=(\\?))\2.)*?\1(?![^\s\)])",
         lambda s, t: (TokenType.QuotedLiteral, t)),
        # Numbers on their own
        (r"(?<![^\s\(])-?[0-9]+(?![^\s\)\(])", lambda s, t: (TokenType.Number,
                                                             t)),
        # Left Paren
        (r"\(", lambda s, t: (TokenType.LeftParen, t)),
        # Right Paren
        (r"\)", lambda s, t: (TokenType.RightParen, t)),
        # Either a valid function name or variable name.
        (r"(?<![^\s\(])[a-zA-z_][a-zA-Z0-9_]*(?![^\s\)\(])",
         lambda s, t: (TokenType.Word, t)),
        # Variable dereference.
        (r"(?<![^\s\(])\${[a-zA-z_][a-zA-Z0-9_]*}(?![^\s\)])",
         lambda s, t: (TokenType.Deref, t)),
        # Newline
        (r"\n", lambda s, t: (TokenType.Newline, t)),
        # Whitespace
        (r"\s+", lambda s, t: (TokenType.Whitespace, t)),
        # The beginning of a double-quoted string, terminating at end of line
        (r"(?<![^\s\(\\])[\"]([^\"]|\\[\"])*$",
         lambda s, t: (TokenType.BeginDoubleQuotedLiteral, t)),
        # The end of a double-quoted string
        (r"[^\s]*(?<!\\)[\"](?![^\s\)])",
         lambda s, t: (TokenType.EndDoubleQuotedLiteral, t)),
        # The beginning of a single-quoted string, terminating at end of line
        (r"(?<![^\s\(\\])[\']([^\']|\\[\'])*$",
         lambda s, t: (TokenType.BeginSingleQuotedLiteral, t)),
        # The end of a single-quoted string
        (r"[^\s]*(?<!\\)[\'](?![^\s\)])",
         lambda s, t: (TokenType.EndSingleQuotedLiteral, t)),
        # Begin-RST Comment Block
        (r"#.rst:$", lambda s, t: (TokenType.BeginRSTComment, t)),
        # Begin Inline RST
        (r"#\[=*\[.rst:$", lambda s, t: (TokenType.BeginInlineRST, t)),
        # End Inline RST
        (r"#\]=*\]$", lambda s, t: (TokenType.EndInlineRST, t)),
        # Comment
        (r"#", lambda s, t: (TokenType.Comment, t)),
        # Catch-all for literals which are compound statements.
        (r"([^\s\(\)]+|[^\s\(]*[^\)]|[^\(][^\s\)]*)",
         lambda s, t: (TokenType.UnquotedLiteral, t))
    ])

    tokens_return = []

    lines = contents.splitlines(True)
    lineno = 0
    for line in lines:
        lineno += 1
        col = 1

        tokens, remaining = scanner.scan(line)
        if remaining != "":
            msg = "Unknown tokens found on line {0}: {1}".format(lineno,
                                                                 remaining)
            raise RuntimeError(msg)

        for token_type, token_contents in tokens:
            tokens_return.append(Token(type=token_type,
                                       content=token_contents,
                                       line=lineno,
                                       col=col))

            col += len(token_contents)

    return tokens_return
297640
def _build_regexes():
    
    # Define a regex that matches ASCII text.
    encoding_regexes = {'ascii': re.compile('^[\x00-\x7f]*$')}

    for encoding in CHARMAP_ENCODINGS:
        # Make a sequence of characters that bytes \x80 to \xFF decode to
        # in each encoding, as well as byte \x1A, which is used to represent
        # the replacement character � in the sloppy-* encodings.
        byte_range = bytes(list(range(0x80, 0x100)) + [0x1a])
        charlist = byte_range.decode(encoding)

        # The rest of the ASCII bytes -- bytes \x00 to \x19 and \x1B
        # to \x7F -- will decode as those ASCII characters in any encoding we
        # support, so we can just include them as ranges. This also lets us
        # not worry about escaping regex special characters, because all of
        # them are in the \x1B to \x7F range.
        regex = '^[\x00-\x19\x1b-\x7f{0}]*$'.format(charlist)
        encoding_regexes[encoding] = re.compile(regex)
    return encoding_regexes
961750
function _tokenize( prop ) {

  var tokens = [],
      patterns,
      src = prop.source;

  src = typeof src !== 'string' ? src : src.split(' ');

  patterns = {
    operator: /^[+\-\*\/\%]$/, // +-*/%
    // @TODO Improve this regex to cover numbers like 1,000,000
    number: /^\$?\d+[\.,]?\d+$/
  };

  function _pushToken( val, type ) {
    var token = {
      value: val,
      type: type
    };
    tokens.push( token );
  }

  // Return early if it's a function.
  if ( typeof src === 'function' ) {
    _pushToken( src, 'function' );
    return tokens;
  }

  // @TODO DRY this up.
  for ( var i = 0, len = src.length; i < len; i++ ) {
    if ( patterns.operator.test(src[i]) ) {
      _pushToken( src[i].match(patterns.operator)[0], 'operator' );
    } else if ( patterns.number.test(src[i]) ) {
      _pushToken( parseFloat( src[i].match(patterns.number)[0] ), 'number' );
    } else {
      _pushToken( src[i], 'name' );
    }
  }
  return tokens;

}
247551
def listified_tokenizer(source):
    """"""
    io_obj = io.StringIO(source)
    return [list(a) for a in tokenize.generate_tokens(io_obj.readline)]
1340794
def _exprparser(
        expr, scope, lang=None, conf=None, configurable=None,
        safe=DEFAULT_SAFE, besteffort=DEFAULT_BESTEFFORT, tostr=False
):
    """"""

    if scope is None:
        scope = {}

    scope.update({
        'configurable': configurable,
        'conf': conf
    })

    expr = REGEX_EXPR_R.sub(
        _refrepl(
            configurable=configurable, conf=conf, safe=safe, scope=scope,
            besteffort=besteffort
        ), expr
    )

    result = resolve(
        expr=expr, name=lang, safe=safe, scope=scope, tostr=tostr,
        besteffort=besteffort
    )

    return result
926321
def _process_regex(cls, regex, rflags, state):
        """"""
        if isinstance(regex, Future):
            regex = regex.get()
        return re.compile(regex, rflags).match
998302
def get_tokens_unprocessed(self, text, stack=('root',)):
        """"""
        self.content_type = None
        return RegexLexer.get_tokens_unprocessed(self, text, stack)
331563
def simple_preprocess(doc, deacc=False, min_len=2, max_len=15):
    
    tokens = [
        token for token in tokenize(doc, lower=True, deacc=deacc, errors='ignore')
        if min_len <= len(token) <= max_len and not token.startswith('_')
    ]
    return tokens
383867
def _compile_tag_re(self):
        
        queue_tag_list = []
        for regex_str, tags in iteritems(self._queue_tag_re):
            try:
                queue_tag_list.append([re.compile(regex_str), [t.strip() for t in tags.split(',')]])
            except TypeError:
                log.warning('{} is not a valid regular expression and will be ignored'.format(regex_str))
        return queue_tag_list
573127
def generate(regex, Ns):
    ""
    return sorted(regex_parse(regex)[0](Ns),
                  key=lambda s: (len(s), s))
810040
def compile(expression, escape_funcs=None, unescape_funcs=None):
    
    return _compile_from_parse_tree(
        parse_regex(tokenize_regex(expression)),
        escape_funcs=escape_funcs,
        unescape_funcs=unescape_funcs)
1055211
def _compileRegExp(string, insensitive, minimal):
        
        flags = 0
        if insensitive:
            flags = re.IGNORECASE

        string = string.replace('[_[:alnum:]]', '[\\w\\d]') # ad-hoc fix for C++ parser
        string = string.replace('[:digit:]', '\\d')
        string = string.replace('[:blank:]', '\\s')

        try:
            return re.compile(string, flags)
        except (re.error, AssertionError) as ex:
            _logger.warning("Invalid pattern '%s': %s", string, str(ex))
            return None
851866
def _proc_token(spec, mlines):
    """"""
    spec = spec.strip().replace(" ", "")
    regexp = re.compile(r".*[^0123456789\-,]+.*")
    tokens = spec.split(",")
    cond = any([not item for item in tokens])
    if ("--" in spec) or ("-," in spec) or (",-" in spec) or cond or regexp.match(spec):
        raise RuntimeError("Argument `lrange` is not valid")
    lines = []
    for token in tokens:
        if token.count("-") > 1:
            raise RuntimeError("Argument `lrange` is not valid")
        if "-" in token:
            subtokens = token.split("-")
            lmin, lmax = (
                int(subtokens[0]),
                int(subtokens[1]) if subtokens[1] else mlines,
            )
            for num in range(lmin, lmax + 1):
                lines.append(num)
        else:
            lines.append(int(token))
    if lines != sorted(lines):
        raise RuntimeError("Argument `lrange` is not valid")
    return lines
60318
private void parseLiteral(String token) {
        Matcher m = Pattern.compile(String.format(ESCAPED_LITERAL_PATTERN_TEMPLATE, escapeCharacter)).matcher(token);
        while (m.find()) {
            cleanedToken = trimSpecialCharacters(m.group(1));
            if (tokenHasContent()) {
                mergedTokens.add(new LiteralToken(cleanedToken));
            }
        }
    }
178103
def get_tokens(source):
    
    if not source.endswith('\n'):
        source += '\n'

    # Cache various valid character sets for speed.
    valid_identifier_first_chars = VALID_IDENTIFIER_FIRST_CHARS
    valid_identifier_chars = VALID_IDENTIFIER_CHARS
    hex_digits = HEX_DIGITS
    int_or_float_digits = INT_OR_FLOAT_DIGITS
    int_or_float_digits2 = int_or_float_digits | set('.')

    # Ignore tokens while in a #if 0 block.
    count_ifs = 0

    i = 0
    end = len(source)
    while i < end:
        # Skip whitespace.
        while i < end and source[i].isspace():
            i += 1
        if i >= end:
            return

        token_type = UNKNOWN
        start = i
        c = source[i]
        # Find a string token.
        if c in valid_identifier_first_chars or c == '_':
            token_type = NAME
            while source[i] in valid_identifier_chars:
                i += 1
            # String and character constants can look like a name if
            # they are something like L"".
            if source[i] == "'" and source[start:i] in _STR_PREFIXES:
                token_type = CONSTANT
                i = _get_char(source, start, i)
            elif source[i] == '"' and source[start:i] in _STR_PREFIXES:
                token_type = CONSTANT
                i = _get_string(source, i)
        elif c == '/' and source[i + 1] == '/':  # Find // comments.
            i = _find(source, '\n', i)
            continue
        elif c == '/' and source[i + 1] == '*':  # Find /* comments. */
            i = _find(source, '*/', i) + 2
            continue
        elif c in '<>':                          # Handle '<' and '>' tokens.
            token_type = SYNTAX
            i += 1
            new_ch = source[i]
            # Do not merge '>>' or '>>=' into a single token
            if new_ch == c and c != '>':
                i += 1
                new_ch = source[i]
            if new_ch == '=':
                i += 1
        elif c in ':+-&|=':                      # Handle 'XX' and 'X=' tokens.
            token_type = SYNTAX
            i += 1
            new_ch = source[i]
            if new_ch == c:
                i += 1
            elif c == '-' and new_ch == '>':
                i += 1
            elif new_ch == '=':
                i += 1
        elif c in '!*^%/':                       # Handle 'X=' tokens.
            token_type = SYNTAX
            i += 1
            new_ch = source[i]
            if new_ch == '=':
                i += 1
        elif c in '()[]{}~?;.,':                 # Handle single char tokens.
            token_type = SYNTAX
            i += 1
            if c == '.' and source[i].isdigit():
                token_type = CONSTANT
                i += 1
                while source[i] in int_or_float_digits:
                    i += 1
                # Handle float suffixes.
                for suffix in ('l', 'f'):
                    if suffix == source[i:i + 1].lower():
                        i += 1
                        break
        elif c.isdigit():                        # Find integer.
            token_type = CONSTANT
            if c == '0' and source[i + 1] in 'xX':
                # Handle hex digits.
                i += 2
                while source[i] in hex_digits:
                    i += 1
            else:
                while source[i] in int_or_float_digits2:
                    i += 1
            # Handle integer (and float) suffixes.
            if source[i].isalpha():
                for suffix in ('ull', 'll', 'ul', 'l', 'f', 'u'):
                    size = len(suffix)
                    if suffix == source[i:i + size].lower():
                        i += size
                        break
        elif c == '"':                           # Find string.
            token_type = CONSTANT
            i = _get_string(source, i)
        elif c == "'":                           # Find char.
            token_type = CONSTANT
            i = _get_char(source, start, i)
        elif c == '#':                           # Find pre-processor command.
            token_type = PREPROCESSOR
            got_if = source[i:i + 3] == '#if'
            if count_ifs and source[i:i + 6] == '#endif':
                count_ifs -= 1
                if count_ifs == 0:
                    source = source[:i].ljust(i + 6) + source[i + 6:]
                    continue

            # Handle preprocessor statements (\ continuations).
            while True:
                i1 = source.find('\n', i)
                i2 = source.find('//', i)
                i3 = source.find('/*', i)
                i4 = source.find('"', i)
                # Get the first important symbol (newline, comment, EOF/end).
                i = min([x for x in (i1, i2, i3, i4, end) if x != -1])

                # Handle comments in #define macros.
                if i == i3:
                    i = _find(source, '*/', i) + 2
                    source = source[:i3].ljust(i) + source[i:]
                    continue

                # Handle #include "dir//foo.h" properly.
                if source[i] == '"':
                    i = _find(source, '"', i + 1) + 1
                    continue

                # Keep going if end of the line and the line ends with \.
                if i == i1 and source[i - 1] == '\\':
                    i += 1
                    continue

                if got_if:
                    begin = source.find('(', start, i)
                    if begin == -1:
                        begin = source.find(' ', start)
                    begin = begin + 1
                    s1 = source.find(' ', begin)
                    s2 = source.find(')', begin)
                    s3 = source.find('\n', begin)
                    s = min([x for x in (s1, s2, s3, end) if x != -1])

                    condition = source[begin:s]
                    if (
                        count_ifs or
                        condition == '0' or
                        condition == '__OBJC__'
                    ):
                        count_ifs += 1
                break
        elif c == '\\':                          # Handle \ in code.
            # This is different from the pre-processor \ handling.
            i += 1
            continue
        elif count_ifs:
            # Ignore bogus code when we are inside an #if block.
            i += 1
            continue
        else:
            raise TokenError("unexpected token '{0}'".format(c))

        if count_ifs:
            continue

        assert i > 0
        yield Token(token_type, source[start:i], start, i)
147936
def _make_regex(self):
    
    return re.compile("|".join(map(re.escape, self.keys())))