Using regex invokes this function, which significantly impacts performance of adapt. it is an N! operation.

        Args:
            tokens(list): list of tokens for Yield results.

        Yields:
            str: ?
430200
def _iterate_subsequences(self, tokens):
        
        for start_idx in xrange(len(tokens)):
            for end_idx in xrange(start_idx + 1, len(tokens) + 1):
                yield ' '.join(tokens[start_idx:end_idx]), start_idx
684165
function make_pram_getter(tokens) {
	return function (results) {
		// Get only the capturing tokens
		var capture_tokens = remove_raw(tokens);
		var res = {};

		// If the regex didn't actually match, just return an empty object
		if (!results) return res;

		// Remove the first item and iterate through the capture groups
		results.slice(1).forEach(function (capture, index) {
			// Retreive the token description for the capture group
			var token = capture_tokens[index];
			var param = capture;
			// If the token doesn't have a name, continue to next group
			if (!token.name) return;

			// If the token is `multi`, split the capture along `/`, remove empty items
			if (token.type === "multi") {
				param = capture.split("/");
				if (!last(param))
					param = remove_last(param);
				// Otherwise, remove any trailing `/`
			} else if (last(capture) === "/")
				param = remove_last(capture);
			// Set the param on the result object
			res[token.name] = param;
		});
		return res;
	}
}
1099156
function regex (strings, ...args) {
  return String.raw(strings, ...args.map(_.escapeRegExp))
}
542088
def get_pairs(self, format_string):
        
        for token in self.get_tokens(format_string):
            match = self.name_pattern.search(token)
            name = match.group(1) if match else None
            yield FormatStringToken(name=name, text=token)
599062
function format (str, tokens) {
  for (let i = 0; i < tokens.length; i++) {
    str = str.replace(tokens[i].token, tokens[i].replacement)
  }
  return str
}
1154411
def matches(self, regex, flags=0):
        
        return self._generate_test(
            lambda value: re.match(regex, value, flags),
            ('matches', self._path, regex)
        )
436293
def collect_tokens(cls, parseresult, mode):
        
        inner = '(%s)' if mode=='parens' else '[%s]'
        if parseresult is None: return []
        tokens = []
        for token in parseresult.asList():
            # If value is a tuple, the token will be a list
            if isinstance(token, list):
                token = cls.recurse_token(token, inner)
                tokens[-1] = tokens[-1] + token
            else:
                if token.strip() == ',': continue
                tokens.append(cls._strip_commas(token))
        return tokens
197891
def _iter_tokens(self):
        """"""
        reobj, actions, nextstates = self._rules[self.states[-1]]
        mobj = reobj.match(self.string, self.pos)
        while mobj is not None:
            text = mobj.group(0)
            idx = mobj.lastindex - 1
            nextstate = nextstates[idx]

            # Take action
            actions[idx](self, text)
            while self.tokens:
                yield self.pop_token()
            if nextstate and nextstate != self.states[-1]:
                self.states[-1] = nextstate

            # Update position variables
            self.pos = mobj.end()
            lines = text.split('\n')
            nlines = len(lines) - 1
            if nlines == 0:
                self.offset = self.offset + len(lines[0])
            else:
                self.lineno = self.lineno + nlines
                self.offset = 1 + len(lines[-1])

            reobj, actions, nextstates = self._rules[self.states[-1]]
            mobj = reobj.match(self.string, self.pos)

        if self.pos != len(self.string):
            msg = "unexpected character"
            text = self.string[self.pos]
            raise RunError(msg, self.lineno, self.offset, text)

        yield EndToken("", self.lineno, self.offset)
1305674
function matcherFromTokens( tokens, context, xml ) {
	var token, matcher,
		i = 0;

	for ( ; (token = tokens[i]); i++ ) {
		if ( Expr.relative[ token.part ] ) {
			matcher = addCombinator( matcher, Expr.relative[ token.part ], context );
		} else {
			token.captures.push( context, xml );
			matcher = addMatcher( matcher, Expr.filter[ token.part ].apply( null, token.captures ) );
		}
	}

	return matcher;
}
61501
def _tokens_from_patsy(node):
    
    for n in node.args:
        for t in _tokens_from_patsy(n):
            yield t

    if node.token:
        yield node.token
57185
def tokenizer(text):
    
    for entry in text.split('$$$$\n'):
        if entry.rstrip():
            lines_stream = deque(entry.split('\n'))
        else:
            continue

        # yield from _molfile(stream=lines_stream)
        for token in _molfile(stream=lines_stream):
            yield token

        if len(lines_stream):
            # yield from _sdfile(stream=lines_stream)
            for token in _sdfile(stream=lines_stream):
                yield token

    yield EndOfFile()
716828
def _MatchDirective(token):
    """"""
    # Tokens below must start with '.'
    if token.startswith('.'):
        token = token[1:]
    else:
        return None, None
    
    if token == 'end':
        return END_TOKEN, None
    
    if token == 'alternates with':
        return ALTERNATES_TOKEN, token
    
    if token.startswith('or'):
        if token.strip() == 'or':
            return OR_TOKEN, None
        else:
            pred_str = token[2:].strip()
            return OR_TOKEN, pred_str
    
    match = _SECTION_RE.match(token)
    if match:
        repeated, section_name = match.groups()
        if repeated:
            return REPEATED_SECTION_TOKEN, section_name
        else:
            return SECTION_TOKEN, section_name
    
    if token.startswith('template '):
        return SUBST_TEMPLATE_TOKEN, token[9:].strip()
    if token.startswith('define '):
        return DEF_TOKEN, token[7:].strip()
    
    if token.startswith('if '):
        return IF_TOKEN, token[3:].strip()
    if token.endswith('?'):
        return PREDICATE_TOKEN, token
    
    return None, None
223991
def token_generator(self, texts, **kwargs):
        
        for text_idx, text in enumerate(texts):
            if self.lower:
                text = text.lower()
            for char in text:
                yield text_idx, char
944040
def tokenize_conf_stream(conf_handle):
    
    for line in conf_handle:
        if line.startswith("#"):
            continue
        tokens = line.split("=")
        if len(tokens) != 2:
            continue
        yield (tokens[0].strip(), tokens[1].strip())
963409
def something(TokenClass):
    """"""
    def callback(lexer, match, context):
        text = match.group()
        if not text:
            return
        yield match.start(), TokenClass, text
        context.pos = match.end()
    return callback
945011
def filter_lines(lines, filter_regex, groups=None):
    
    pattern = re.compile(filter_regex)
    for line in lines:
        match = pattern.search(line)
        if match:
            if groups is None:
                yield line
            elif len(groups) == 1:
                yield match.group(groups[0])
            else:
                matched_groups = match.groupdict()
                yield tuple(matched_groups.get(group) for group in groups)
173905
def parse_regex(regex_tokens):
    
    # We add a closing brace because that represents the final pop of the stack.
    tokens = [')'] + regex_tokens[::-1]

    def wrap(lst):
        """ Turn list into sequence when it contains several items. """
        if len(lst) == 1:
            return lst[0]
        else:
            return Sequence(lst)

    def _parse():
        or_list = []
        result = []

        def wrapped_result():
            if or_list == []:
                return wrap(result)
            else:
                or_list.append(result)
                return Any([wrap(i) for i in or_list])

        while tokens:
            t = tokens.pop()

            if t.startswith('(?P<'):
                variable = Variable(_parse(), varname=t[4:-1])
                result.append(variable)

            elif t in ('*', '*?'):
                greedy = (t == '*')
                result[-1] = Repeat(result[-1], greedy=greedy)

            elif t in ('+', '+?'):
                greedy = (t == '+')
                result[-1] = Repeat(result[-1], min_repeat=1, greedy=greedy)

            elif t in ('?', '??'):
                if result == []:
                    raise Exception('Nothing to repeat.' + repr(tokens))
                else:
                    greedy = (t == '?')
                    result[-1] = Repeat(result[-1], min_repeat=0, max_repeat=1, greedy=greedy)

            elif t == '|':
                or_list.append(result)
                result = []

            elif t in ('(', '(?:'):
                result.append(_parse())

            elif t == '(?!':
                result.append(Lookahead(_parse(), negative=True))

            elif t == '(?=':
                result.append(Lookahead(_parse(), negative=False))

            elif t == ')':
                return wrapped_result()

            elif t.startswith('#'):
                pass

            elif t.startswith('{'):
                # TODO: implement!
                raise Exception('{}-style repitition not yet supported' % t)

            elif t.startswith('(?'):
                raise Exception('%r not supported' % t)

            elif t.isspace():
                pass
            else:
                result.append(Regex(t))

        raise Exception("Expecting ')' token")

    result = _parse()

    if len(tokens) != 0:
        raise Exception("Unmatched parantheses.")
    else:
        return result