Sets up compiled regex objects for parsing code elements.
960378
def setup_regex(self):
        """"""
        #Regex for extracting modules from the code
        self._RX_MODULE = r"(\n|^)\s*module\s+(?P<name>[a-z0-9_]+)(?P<contents>.+?)end\s*module"
        self.RE_MODULE = re.compile(self._RX_MODULE, re.I | re.DOTALL)
        self._RX_PROGRAM = r"(\n|^)\s*program\s+(?P<name>[a-z0-9_]+)(?P<contents>.+?)end\s*program"
        self.RE_PROGRAM = re.compile(self._RX_PROGRAM, re.I | re.DOTALL)

        #Regex for use statements in a module
        self._RX_USE = r"^\s*use\s+(?P<name>[^,]+?)(\s*,\s+only\s*:(?P<only>[A-Za-z0-9_\s,]+?))?$"
        self.RE_USE = re.compile(self._RX_USE, re.I | re.M)
        #Regex for finding if the module is private
        self._RX_PRIV = "private.+?(type|contains)"
        self.RE_PRIV = re.compile(self._RX_PRIV, re.DOTALL | re.I)
        #Regex for finding publcily labeled members declared using public keyword.
        self._RX_PUBLIC = r"\n\s*public\s+(?P<methods>[A-Za-z0-9_,\s&\n]+)"
        self.RE_PUBLIC = re.compile(self._RX_PUBLIC, re.I)
        #Regex for finding text before type or contains declarations that may contian members.
        self._RX_MEMBERS = "(?P<preamble>.+?)(\s+type[,\s]|contains)"
        self.RE_MEMBERS = re.compile(self._RX_MEMBERS, re.DOTALL | re.I)

        self._RX_PRECOMP = r"#endif"
        self.RE_PRECOMP = re.compile(self._RX_PRECOMP, re.I)
383867
def _compile_tag_re(self):
        
        queue_tag_list = []
        for regex_str, tags in iteritems(self._queue_tag_re):
            try:
                queue_tag_list.append([re.compile(regex_str), [t.strip() for t in tags.split(',')]])
            except TypeError:
                log.warning('{} is not a valid regular expression and will be ignored'.format(regex_str))
        return queue_tag_list
951017
public void insertStopRegexes(String... regexes) {
        for (String regex : regexes) {
            try {
                regexList.add(Pattern.compile(regex));
            } catch (Exception e) {
                e.printStackTrace();
                LOG.error("regex err : " + regex, e);
            }
        }

    }
1053250
def _lazy_re_compile(regex, flags=0):
    """"""
    def _compile():
        # Compile the regex if it was not passed pre-compiled.
        if isinstance(regex, str):
            return re.compile(regex, flags)
        else:
            assert not flags, "flags must be empty if regex is passed pre-compiled"
            return regex
    return SimpleLazyObject(_compile)
742347
def _create_regex_if_none(self):
        
        if self._regex is None:
            self._regex = re.compile(self._pattern, re.UNICODE)
1128976
def compile_matchers(formatcode):
    
    matchers = {}
    for node_type in ["leaf", "single", "internal"]:
        if node_type == "leaf" or node_type == "single":
            container1 = NW_FORMAT[formatcode][0][0]
            container2 = NW_FORMAT[formatcode][1][0]
            converterFn1 = NW_FORMAT[formatcode][0][1]
            converterFn2 = NW_FORMAT[formatcode][1][1]
            flexible1 = NW_FORMAT[formatcode][0][2]
            flexible2 = NW_FORMAT[formatcode][1][2]
        else:
            container1 = NW_FORMAT[formatcode][2][0]
            container2 = NW_FORMAT[formatcode][3][0]
            converterFn1 = NW_FORMAT[formatcode][2][1]
            converterFn2 = NW_FORMAT[formatcode][3][1]
            flexible1 = NW_FORMAT[formatcode][2][2]
            flexible2 = NW_FORMAT[formatcode][3][2]

        if converterFn1 == str:
            FIRST_MATCH = "("+_NAME_RE+")"
        elif converterFn1 == float:
            FIRST_MATCH = "("+_FLOAT_RE+")"
        elif converterFn1 is None:
            FIRST_MATCH = '()'

        if converterFn2 == str:
            SECOND_MATCH = "(:"+_NAME_RE+")"
        elif converterFn2 == float:
            SECOND_MATCH = "(:"+_FLOAT_RE+")"
        elif converterFn2 is None:
            SECOND_MATCH = '()'

        if flexible1 and node_type != 'leaf':
            FIRST_MATCH += "?"
        if flexible2:
            SECOND_MATCH += "?"

        matcher_str= '^\s*%s\s*%s\s*(%s)?\s*$' % (FIRST_MATCH, SECOND_MATCH, _NHX_RE)
        compiled_matcher = re.compile(matcher_str)
        matchers[node_type] = [container1, container2, converterFn1, converterFn2, compiled_matcher]

    return matchers
1008618
def set_code_exprs(self, codes):
        """"""
        self.code_objs = dict()
        self._codes = []
        for code in codes:
            self.append_code_expr(code)
1015865
def _parse_code(self, line):
        
        log.debug("*** parse CODE: >>>%r<<<", line)
        parts = self.regex_split_all.split(line, maxsplit=1)
        if len(parts) == 1:  # end
            self.line_data.append(BASIC_Code(parts[0]))
            return
        pre, match, post = parts
        log.debug("\tpre: >>>%r<<<", pre)
        log.debug("\tmatch: >>>%r<<<", match)
        log.debug("\tpost: >>>%r<<<", post)

        if match == '"':
            log.debug("%r --> parse STRING", match)
            self.line_data.append(BASIC_Code(pre))
            string_part, rest = self._parse_string(post)
            self.line_data.append(BASIC_String(match + string_part))
            if rest:
                self._parse_code(rest)
            return

        self.line_data.append(BASIC_Code(pre + match))

        if match == "DATA":
            log.debug("%r --> parse DATA", match)
            data_part, rest = self._parse_data(post)
            self.line_data.append(BASIC_Data(data_part))
            if rest:
                self._parse_code(rest)
            return
        elif match in ("'", "REM"):
            log.debug("%r --> consume rest of the line as COMMENT", match)
            if post:
                self.line_data.append(BASIC_Comment(post))
            return

        raise RuntimeError("Wrong Reg.Exp.? match is: %r" % match)
1055211
def _compileRegExp(string, insensitive, minimal):
        
        flags = 0
        if insensitive:
            flags = re.IGNORECASE

        string = string.replace('[_[:alnum:]]', '[\\w\\d]') # ad-hoc fix for C++ parser
        string = string.replace('[:digit:]', '\\d')
        string = string.replace('[:blank:]', '\\s')

        try:
            return re.compile(string, flags)
        except (re.error, AssertionError) as ex:
            _logger.warning("Invalid pattern '%s': %s", string, str(ex))
            return None
532175
public function setupRules()
    {
        $standard = new Validator(['!string', '!comment']);

        $this->rules->addMany([
            'keyword' => new Rule(new WordMatcher([
                'and', 'del', 'from', 'not', 'while', 'as', 'elif', 'global', 'or', 'with', 'assert', 'else', 'if',
                'pass', 'yield', 'break', 'except', 'import', 'print', 'class', 'exec', 'in', 'raise', 'continue',
                'finally', 'is', 'return', 'def', 'for', 'lambda', 'try',
            ])),

            'operator' => new Rule(
                new RegexMatcher('/([-+%=]=?|!=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~])|\b(or|and|not)\b/'), [
                    'priority' => -1
                ]
            ),

            'expression' => new Rule(new RegexMatcher('/\{(\S+)\}/'), [
                'context' => ['string']
            ]),

            'variable' => [
                new Rule(new RegexMatcher('/[^\w.]([a-z_]\w*)\.\w/i')),
                'property' => new Rule(new RegexMatcher('/(?=(?:\w|\)|\])\s*\.([a-z_]\w*))/i'), [
                    'priority' => -2,
                    'context' => ['*none', '*expression']
                ]),
            ],

            'symbol' => [
                new Rule(new RegexMatcher('/import\s+([a-z_][\w.]*)(?:\s*,\s*([a-z_][\w.]*))*/i', [
                    1 => Token::NAME,
                    2 => Token::NAME,
                ])),
                'library' => new Rule(new RegexMatcher('/from\s+([a-z_][\w.]*)\s+import/i', [
                    1 => Token::NAME,
                ]))
            ],

            'operator.escape' => new Rule(new RegexMatcher('/(\\\(?:\R|.|[0-7]{3}|x\x{2}))/s'), [
                'context' => Validator::everywhere()
            ]),

            'comment' => new Rule(new CommentMatcher(['#'])),
            'constant.special' => [
                new Rule(new WordMatcher(['True', 'False', 'NotImplemented', 'Ellipsis'], [
                    'case-sensitivity' => true
                ])),
                new Rule(new RegexMatcher('/\b(__\w+__)\b/'))
            ],
            'call' => new Rule(new RegexMatcher('/([a-z_]\w*)\s*\(/i'), ['priority' => 2]),

            'meta.newline' => new CloseRule(new RegexMatcher('/()\R/'), [
                'factory' => new TokenFactory(TerminatorToken::class),
                'context' => ['!operator.escape', 'string.single-line'],
                'closes'  => ['string.single-line.double', 'string.single-line.single']
            ]),

            'number' => new Rule(
                new RegexMatcher('/(-?\b(?:0[bo])?(?:(?:\d|0x[\da-f])[\da-f]*\.?\d*|\.\d+)(?:e[+-]?\d+)?j?)\b/')
            ),

            'string' => [
                'single-line' => [
                    'double' => new Rule(new RegexMatcher('/(?:^|[^"])(")(?=[^"]|$)/'), [
                        'factory' => new TokenFactory(ContextualToken::class),
                        'context' => $standard,
                    ]),
                    'single' => new Rule(new RegexMatcher('/(?:^|[^\'])(\')(?=[^\']|$)/'), [
                        'factory' => new TokenFactory(ContextualToken::class),
                        'context' => $standard,
                    ]),
                ],
                'multi-line' => [
                    'double' => new Rule(new SubStringMatcher('"""'), [
                        'factory'  => new TokenFactory(ContextualToken::class),
                        'context'  => $standard,
                        'priority' => 2,
                    ]),
                    'single' => new Rule(new SubStringMatcher('\'\'\''), [
                        'factory'  => new TokenFactory(ContextualToken::class),
                        'context'  => $standard,
                        'priority' => 2,
                    ]),
                ]
            ],

            'symbol.function' => new Rule(new RegexMatcher('/def\s+([a-z_]\w+)\s*\(/i')),
            'symbol.class'    => new Rule(new RegexMatcher('/class\s+([a-z_]\w+)/i')),
        ]);
    }
220085
def tokenize(code):
    """"""
    tok_regex = '|'.join('(?P<{}>{})'.format(*pair) for pair in _tokens)
    tok_regex = re.compile(tok_regex, re.IGNORECASE|re.M)
    line_num = 1
    line_start = 0
    for mo in re.finditer(tok_regex, code):
        kind = mo.lastgroup
        value = mo.group(kind)
        if kind == 'NEWLINE':
            line_start = mo.end()
            line_num += 1
        elif kind == 'SKIP' or value=='':
            pass
        else:
            column = mo.start() - line_start
            yield Token(kind, value, line_num, column)
799239
def compile!
      regexp_string = ''
      @formats.inject(0) { |index, format_string|
        format = Format.new(format_string).compile!
        @formats_hash[format_string] = format
        @match_indexes[index] = format
        regexp_string = "#{regexp_string}(#{format.regexp_string})|"
        index + format.token_count + 1 # add one for wrapper capture
      }
      @regexp = %r[\A(?:#{regexp_string.chop})\z]
      self
    end
1325790
def _make_regex(self):
        
        rxp = "|".join(
            map(self._address_rxp, self.keys()),
        )
        self._regex = re.compile(
            rxp,
            re.IGNORECASE,
        )
500172
def _regex_span(_regex, _str, case_insensitive=True):
    
    if case_insensitive:
        flags = regex.IGNORECASE | regex.FULLCASE | regex.VERSION1
    else:
        flags = regex.VERSION1
    comp = regex.compile(_regex, flags=flags)
    matches = comp.finditer(_str)
    for match in matches:
        yield match
122072
def register_regex_entity(self, regex_str):
        
        if regex_str and regex_str not in self._regex_strings:
            self._regex_strings.add(regex_str)
            self.regular_expressions_entities.append(re.compile(regex_str, re.IGNORECASE))
352485
public void addRegExp(RegExpLiteral re) {
        if (re == null) codeBug();
        if (regexps == null)
            regexps = new ArrayList<RegExpLiteral>();
        regexps.add(re);
        re.putIntProp(REGEXP_PROP, regexps.size() - 1);
    }
1202224
def _starts_with_regex(line, regex):
    """"""
    if not regex.startswith('^'):
        regex = '^' + regex
    reg = re.compile(regex)
    return reg.match(line)