Add constructed spacy rule to Matcher
966129
def _load_matcher(self) -> None:
        
        for id_key in self._rule_lst:
            if self._rule_lst[id_key].active:
                pattern_lst = [a_pattern.spacy_token_lst for a_pattern in self._rule_lst[id_key].patterns]

                for spacy_rule_id, spacy_rule in enumerate(itertools.product(*pattern_lst)):
                    self._matcher.add(self._construct_key(id_key, spacy_rule_id), None, list(spacy_rule))
273309
def setup_regex(self):
        """"""
        self._RX_INTERFACE = (r"\n\s*interface\s+(?P<name>[a-z0-9_]+)(\s\((?P<symbol>[.\w+*=/-]+)\))?"
                              r"(?P<contents>.+?)"
                              r"end\s*interface\s+(?P=name)?")
        self.RE_INTERFACE = re.compile(self._RX_INTERFACE, re.I | re.DOTALL)
709943
def prefix(tokens, operator_table):
    """"""
    operator, matched_tokens = operator_table.prefix.match(tokens)
    if operator:
        return TokenMatch(operator, None, matched_tokens)
820036
def custom_tokenizer(self) -> spacyTokenizer:
        
        prefix_re = re.compile(r'''^[\[()\-.,@#$%^&*?|<~+_:;>!"']''')
        infix_re = re.compile(r'''[\[()\-,@#$%^&*?|<~+_:;>!"']|(?<![0-9])\.|\.(?![0-9])|\n+ ''')
        return spacyTokenizer(self.nlp.vocab, rules=None, prefix_search=prefix_re.search, suffix_search=None,
                              infix_finditer=infix_re.finditer, token_match=None)
1363737
def _tokenize(self, comps):
        """"""
        ps = []
        for comp in comps:
            ps.extend([c.strip(' ,') for c in re.split(r'\s+(?=[^{}]*(?:\{|$))', comp)])
        return [p for p in ps if p]
916925
def matched(self, key, regex, ignore_case=False, multi_line=False):
        
        if not isinstance(regex, six.string_types):
            raise TypeError('matched only accept str or unicode')
        self._add_condition(key, '$regex', regex)
        modifiers = ''
        if ignore_case:
            modifiers += 'i'
        if multi_line:
            modifiers += 'm'
        if modifiers:
            self._add_condition(key, '$options', modifiers)
        return self
479477
function matchAt(content, index, match) {
  return content.substring(index, index + match.length) === match;
}
129069
def combine_or(matcher, *more_matchers):
    """"""

    def matcher(cause):
        for sub_matcher in itertools.chain([matcher], more_matchers):
            cause_cls = sub_matcher(cause)
            if cause_cls is not None:
                return cause_cls
        return None

    return matcher
901112
public function setupRules()
    {
        $this->rules->addMany([
            'call.symbol' => new Rule(new RegexMatcher('/(\\\[a-z@]+)/si'), [
                'context' => ['!delimiter.environment'],
                'priority' => -1
            ]),

            'expression.math' => [
                new Rule(new RegexMatcher('/((\${1,2}).*?\2)/s')),
                new Rule(new RegexMatcher('/(\\\\\(.*?\\\\\))/s')),
                new Rule(new RegexMatcher('/(\\\\\[.*?\\\\\])/s')),
                new Rule(
                    new RegexMatcher(
                        '/\\\begin{((?:' . implode('|', self::$mathEnvironments) . ')\*?)}(.*?)\\\end{\1}/s',
                        [2 => Token::NAME]
                    )
                )
            ],

            'delimiter.math' => new Rule(new WordMatcher(['$', '\[', '\]', '\(', '\)'], ['separated' => false]), [
                'context' => ['expression.math']
            ]),

            'symbol.annotation'    => new Rule(new RegexMatcher('/\[(.*?)\]/')), // make argument parsing?
            'symbol.environment' => new Rule(new RegexMatcher('/(\\\(?:begin|end)){(.*?)}/', [
                1 => 'delimiter.environment',
                2 => Token::NAME
            ]), ['context' => Validator::everywhere()]),

            'symbol.label' => new Rule(new RegexMatcher('/\\\(?:label|ref){(.*?)}/')),

            'operator' => [
                new Rule(new WordMatcher(['*', '&', '@', '\\\\'], ['separated' => false]), ['context' => Validator::everywhere()]),
                new Rule(new WordMatcher(['=', '-', '+', '/', '^', '_'], ['separated' => false]), [
                    'context'  => ['expression.math'],
                    'priority' => -1
                ]),
                'punctuation.brackets' => new Rule(new WordMatcher(['{', '}', '[', ']', '(', ')'], ['separated' => false]), [
                    'context' => ['expression.math']
                ]),
                'escape' => new Rule(new RegexMatcher('/(\\\[%@&])/si'), ['context' => Validator::everywhere(), 'priority' => -1])

            ],
            'comment' => new Rule(new CommentMatcher(['%'], [])),

            'format.bold' => new Rule(new RegexMatcher('/\\\textbf({((?>[^{}]+|(?1))+)})/si', [
                2 => Token::NAME
            ])),
            'format.italics' => new Rule(new RegexMatcher('/\\\textit({((?>[^{}]+|(?1))+)})/si', [
                2 => Token::NAME
            ])),

            'variable' => new Rule(new RegexMatcher('/(#\d+)/')),

            # math mode
            'number' => new Rule(new RegexMatcher('/(-?(?:0[0-7]+|0[xX][0-9a-fA-F]+|0b[01]+|[\d,.]+))/'), [
                'context'  => ['expression.math'],
                'priority' => -2
            ]),
        ]);
    }
1288254
def infix(tokens, operator_table):
    """"""
    operator, matched_tokens = operator_table.infix.match(tokens)
    if operator:
        return TokenMatch(operator, None, matched_tokens)
1073627
def _setup_regex(self):
        """"""
        self.RE_COMMENTS = cache.RE_COMMENTS
        self.RE_MODULE = cache.RE_MODULE
        self.RE_TYPE = cache.RE_TYPE
        self.RE_EXEC = cache.RE_EXEC
        self.RE_MEMBERS = cache.RE_MEMBERS
        self.RE_DEPEND = cache.RE_DEPEND
323894
def pretty_print_match(match, parameterized=True):
    """"""
    left_curly = '{{' if parameterized else '{'
    right_curly = '}}' if parameterized else '}'
    match = remove_custom_formatting(match)
    parts = re.split('({}|{})'.format(left_curly, right_curly), match)

    inside_braces = False
    indent_size = 4
    indent = ' ' * indent_size

    output = [parts[0]]
    for current_index, current_part in enumerate(parts[1:]):
        if current_part == left_curly:
            if inside_braces:
                raise AssertionError(u'Found open-braces pair while already inside braces: '
                                     u'{} {} {}'.format(current_index, parts, match))
            inside_braces = True
            output.append(current_part + '\n')
        elif current_part == right_curly:
            if not inside_braces:
                raise AssertionError(u'Found close-braces pair while not inside braces: '
                                     u'{} {} {}'.format(current_index, parts, match))
            inside_braces = False
            output.append(current_part)
        else:
            if not inside_braces:
                stripped_part = current_part.lstrip()
                if stripped_part.startswith('.'):
                    # Strip whitespace before traversal steps.
                    output.append(stripped_part)
                else:
                    # Do not strip whitespace before e.g. the RETURN keyword.
                    output.append(current_part)
            else:
                # Split out the keywords, initially getting rid of commas.
                separate_keywords = re.split(', ([a-z]+:)', current_part)

                # The first item in the separated list is the full first "keyword: value" pair.
                # For every subsequent item, the keyword and value are separated; join them
                # back together, outputting the comma, newline and indentation before them.
                output.append(indent + separate_keywords[0].lstrip())
                for i in six.moves.xrange(1, len(separate_keywords) - 1, 2):
                    output.append(',\n{indent}{keyword} {value}'.format(
                        keyword=separate_keywords[i].strip(),
                        value=separate_keywords[i + 1].strip(),
                        indent=indent))
                output.append('\n')

    return ''.join(output).strip()
816085
function( term ) {
        var split_term = term.split(' ');
        var matchers = [];
        for (var i=0; i < split_term.length; i++) {
          if ( split_term[i].length > 0 ) {
            var matcher = {};
            matcher['partial'] = new RegExp( $.ui.autocomplete.escapeRegex( split_term[i] ), "i" );
            if ( context.settings['relevancy-sorting'] ) {
              matcher['strict'] = new RegExp( "^" + $.ui.autocomplete.escapeRegex( split_term[i] ), "i" );
            }
            matchers.push( matcher );
          }
        };

        return $.grep( context.options, function( option ) {
          var partial_matches = 0;
          if ( context.settings['relevancy-sorting'] ) {
            var strict_match = false;
            var split_option_matches = option.matches.split(' ');
          }
          for ( var i=0; i < matchers.length; i++ ) {
            if ( matchers[i]['partial'].test( option.matches ) ) {
              partial_matches++;
            }
            if ( context.settings['relevancy-sorting'] ) {
              for (var q=0; q < split_option_matches.length; q++) {
                if ( matchers[i]['strict'].test( split_option_matches[q] ) ) {
                  strict_match = true;
                  break;
                }
              };
            }
          };
          if ( context.settings['relevancy-sorting'] ) {
            var option_score = 0;
            option_score += partial_matches * context.settings['relevancy-sorting-partial-match-value'];
            if ( strict_match ) {
              option_score += context.settings['relevancy-sorting-strict-match-value'];
            }
            option_score = option_score * option['relevancy-score-booster'];
            option['relevancy-score'] = option_score;
          }
          return (!term || matchers.length === partial_matches );
        });
      }
550044
function(matches, index, tokens, regexValue) {
      var i = tokens.length;

      while (--i >= 0) {
        var token = tokens[i];
        var type = token.type;

        if (type === _Comment ||
            type === _WhiteSpace ||
            type === _LineTerminator) {
          continue;
        }

        var value = token.value;
        if (type === _Punctuator) {
          if (value === ')') {
            if (this.isValidRegExpPrefix(tokens, i + 1)) {
              break;
            }
          } else if (regexPrefixRe.test(value)) {
            break;
          }
        } else if ((type === _Keyword && regexPrefixRe.test(value)) ||
                   (type === _Template && value.slice(-2) === '${')) {
          break;
        }

        var parts = regexValue.match(tokenizeNotRegExpRe);
        splice.apply(matches, [index, 1].concat(parts));
        return true;
      }
      return false;
    }
984004
def _simplify(self, pattern):
        # type: (str) -> str
        
        """
        # remove optional params
        # TODO(dcramer): it'd be nice to change these into [%s] but it currently
        # conflicts with the other rules because we're doing regexp matches
        # rather than parsing tokens
        result = self._optional_group_matcher.sub(lambda m: "%s" % m.group(1), pattern)

        # handle named groups first
        result = self._named_group_matcher.sub(lambda m: "{%s}" % m.group(1), result)

        # handle non-named groups
        result = self._non_named_group_matcher.sub("{var}", result)

        # handle optional params
        result = self._either_option_matcher.sub(lambda m: m.group(1), result)

        # clean up any outstanding regex-y characters.
        result = (
            result.replace("^", "")
            .replace("$", "")
            .replace("?", "")
            .replace("//", "/")
            .replace("\\", "")
        )

        return result
216382
function concatRegexes (regexes) {
  var rconcat = ''

  if (Array.isArray(regexes)) {
    for (var i = 0, l = regexes.length; i < l; i++) {
      rconcat += regexes[i].source + '|'
    }

    rconcat = rconcat.substr(0, rconcat.length - 1)

    return new RegExp(rconcat)
  }
}
961750
function _tokenize( prop ) {

  var tokens = [],
      patterns,
      src = prop.source;

  src = typeof src !== 'string' ? src : src.split(' ');

  patterns = {
    operator: /^[+\-\*\/\%]$/, // +-*/%
    // @TODO Improve this regex to cover numbers like 1,000,000
    number: /^\$?\d+[\.,]?\d+$/
  };

  function _pushToken( val, type ) {
    var token = {
      value: val,
      type: type
    };
    tokens.push( token );
  }

  // Return early if it's a function.
  if ( typeof src === 'function' ) {
    _pushToken( src, 'function' );
    return tokens;
  }

  // @TODO DRY this up.
  for ( var i = 0, len = src.length; i < len; i++ ) {
    if ( patterns.operator.test(src[i]) ) {
      _pushToken( src[i].match(patterns.operator)[0], 'operator' );
    } else if ( patterns.number.test(src[i]) ) {
      _pushToken( parseFloat( src[i].match(patterns.number)[0] ), 'number' );
    } else {
      _pushToken( src[i], 'name' );
    }
  }
  return tokens;

}