Lex a string.
        
        :param string: The input to lex. 
        :return: A generator which will continue yielding tokens until the end
                 of the input is reached.
198933
def lex(self, string):
        
        self.lexer.input(string)
        for token in self.lexer:
            yield token
899662
def tree(string, token=[WORD, POS, CHUNK, PNP, REL, ANCHOR, LEMMA]):
    
    return Text(string, token)
753616
def tokenize(self, string):
        ''''''
        pos = 0
        while pos < len(string):
            m = SKIP_RE.match(string, pos=pos)
            if m:
                skip_lines = m.group(0).split('\n')
                if len(skip_lines) > 1:
                    self.row += len(skip_lines) - 1
                    self.column = 1 + len(skip_lines[-1])
                else:
                    self.column += len(skip_lines[0])

                pos = m.end()
                continue

            for cls, type, regex in self.token_map:
                m = regex.match(string, pos=pos)
                if m:
                    yield cls(type, m.group(0),
                              self.filename, self.row, self.column)
                    self.column += len(m.group(0))
                    pos = m.end()
                    break
            else:
                raise ConfigParseError(
                    "Couldn't load config in %r row %d, column %d: %r" %
                    (self.filename, self.row, self.column,
                     string[pos:pos+20]))
1258311
def _skip_char_around(self, string, char='\n'):
        
        starts, ends = '', ''
        n = len(char)
        if string.startswith(char):
            starts = string[:n]
            string = string[n:]
        if string.endswith(char):
            ends = string[-n:]
            string = string[:-n]
        string = self._base_compile(string)
        if starts:
            string = starts + string
        if ends:
            string = string + ends
        return string
381940
def _find(string, sub_string, start_index):
    
    result = string.find(sub_string, start_index)
    if result == -1:
        raise TokenError("expected '{0}'".format(sub_string))
    return result
860136
def lex(self, text, start=0):
        
        max = len(text)
        eaten = start
        s = self.state
        r = self.regexes
        toks = self.toks
        while eaten < max:
            for match in r[s].finditer(text, eaten):
                name = match.lastgroup
                tok = toks[name]
                toktext = match.group(name)
                eaten += len(toktext)
                yield (tok.name, toktext)

                if tok.next:
                    s = tok.next
                    break

        self.state = s
1354571
def tokenize(string):
    """"""
    for match in TOKENS_REGEX.finditer(string):
        yield Token(match.lastgroup, match.group().strip(), match.span())
440921
def tokenize_wrapper(input):
    """"""
    skip = set((token.NEWLINE, token.INDENT, token.DEDENT))
    tokens = tokenize.generate_tokens(StringIO.StringIO(input).readline)
    for quintuple in tokens:
        type, value, start, end, line_text = quintuple
        if type not in skip:
            yield quintuple
758604
def str_lexer(mode):
    
    cast_to_const = ConstStrPool.cast_to_const

    def f_raw(inp_str, pos):
        return cast_to_const(mode) if inp_str.startswith(mode, pos) else None

    def f_collection(inp_str, pos):
        for each in mode:
            if inp_str.startswith(each, pos):
                return cast_to_const(each)
        return None

    if isinstance(mode, str):
        return f_raw

    if len(mode) is 1:
        mode = mode[0]
        return f_raw

    return f_collection
837137
def string(s):
    ''''''
    @Parser
    def string_parser(text, index=0):
        slen, tlen = len(s), len(text)
        if text[index:index + slen] == s:
            return Value.success(index + slen, s)
        else:
            matched = 0
            while matched < slen and index + matched < tlen and text[index + matched] == s[matched]:
                matched = matched + 1
            return Value.failure(index + matched, s)
    return string_parser
1332187
public final void mSTRING() throws RecognitionException {
		try {
			int _type = STRING;
			int _channel = DEFAULT_TOKEN_CHANNEL;
			// druidG.g:601:9: ( ( 'STRING' ) )
			// druidG.g:601:10: ( 'STRING' )
			{
			// druidG.g:601:10: ( 'STRING' )
			// druidG.g:601:11: 'STRING'
			{
			match("STRING"); 

			}

			}

			state.type = _type;
			state.channel = _channel;
		}
		finally {
			// do for sure before leaving
		}
	}
714712
def convert(string, lineno):
    
    def parse(string):
        try:
            return parso.parse(string, error_recovery=False,
                               version=os.getenv('F2FORMAT_VERSION', PARSO_VERSION[-1]))
        except parso.ParserSyntaxError as error:
            message = '%s: <%s: %r> from %r' % (error.message, err.error_leaf.token_type,
                                                 err.error_leaf.value, string)
            raise ConvertError(message)

    source = strarray(string)       # strarray source (mutable)
    f_string = [list()]             # [[token, ...], [...], ...] -> concatenable strings

    str_flag = False    # if previous item is token.STRING
    for token in tokenize.generate_tokens(io.StringIO(string).readline):
        cat_flag = False                    # if item is concatenable with previous item, i.e. adjacent string
        if token.type == tokenize.STRING:
            if str_flag:
                cat_flag = True
            if cat_flag:
                f_string[-1].append(token)
            else:
                f_string.append([token])
            str_flag = True
        elif token.type == tokenize.NL:     # skip token.NL
            continue
        else:                               # otherwise, not concatenable
            str_flag = False
        # print(token) ###

    # print() ###
    # import pprint ###
    # pprint.pprint(f_string) ###
    # print() ###

    for tokens in reversed(f_string):   # for each string concatenation
        # check if has f-string literal in this concatenation
        future = any(map(lambda token: re.match(r'^(f|rf|fr)', token.string, re.IGNORECASE), tokens))
        if not future:
            continue

        entryl = list()
        for token in tokens:            # for each token in concatenation
            token_string = token.string

            module = parse(token_string)            # parse AST, get parso.python.tree.Module, _.children -> list
                                                    # _[0] -> parso.python.tree.PythonNode
                                                    # _[1] -> parso.python.tree.EndMarker
            tmpval = module.children[0]             # parsed string token
            tmpent = list()                         # temporary entry list

            if tmpval.type == 'fstring':            # parso.python.tree.PythonNode.type -> str, string / fstring
                # parso.python.tree.PythonNode.children[0] -> parso.python.tree.FStringStart, regex: /^((f|rf|fr)('''|'|"""|"))/
                # parso.python.tree.PythonNode.children[-1] -> parso.python.tree.FStringEnd, regex: /('''|'|"""|")$/
                for obj in tmpval.children[1:-1]:               # traverse parso.python.tree.PythonNode.children -> list # noqa
                    if obj.type == 'fstring_expr':                      # expression part (in braces), parso.python.tree.PythonNode # noqa
                        obj_children = obj.children                             # parso.python.tree.PythonNode.children -> list
                                                                                # _[0] -> parso.python.tree.Operator, '{' # noqa
                                                                                # _[1] -> %undetermined%, expression literal (f_expression) # noqa
                                                                                # _[2] -> %optional%, parso.python.tree.PythonNode, format specification (format_spec) # noqa
                                                                                # -[3] -> parso.python.tree.Operator, '}' # noqa
                        start_expr = obj_children[1].start_pos[1]
                        end_expr = obj_children[1].end_pos[1]
                        tmpent.append(slice(start_expr, end_expr))              # entry of expression literal (f_expression)

                        if obj_children[2].type == 'fstring_format_spec':
                            for node in obj_children[2].children:               # traverse format specifications (format_spec)
                                if node.type == 'fstring_expr':                         # expression part (in braces), parso.python.tree.PythonNode # noqa
                                    node_chld = node.children                                   # parso.python.tree.PythonNode.children -> list # noqa
                                                                                                # _[0] -> parso.python.tree.Operator, '{' # noqa
                                                                                                # _[1] -> %undetermined%, expression literal (f_expression) # noqa
                                                                                                # _[2] -> parso.python.tree.Operator, '}' # noqa
                                    start = node_chld[1].start_pos[1]
                                    end = node_chld[1].end_pos[1]
                                    tmpent.append(slice(start, end))
                    # print('length:', length, '###', token_string[:length], '###', token_string[length:]) ###
            entryl.append((token, tmpent))          # each token with a concatenation entry list

        # print('entry: ', end='') ###
        # pprint.pprint(entryl) ###
        # print() ###

        expr = list()
        for token, entries in entryl:   # extract expressions
            # print(token.string, entries) ###
            for entry in entries:       # walk entries
                temp_expr = token.string[entry]                                 # original expression
                val = parse(temp_expr).children[0]                              # parse AST
                if val.type == 'testlist_star_expr' and \
                        re.fullmatch(r'\(.*\)', temp_expr, re.DOTALL) is None:  # if expression is implicit tuple
                    real_expr = '(%s)' % temp_expr                              # add parentheses
                else:
                    real_expr = temp_expr                                       # or keep original
                expr.append(real_expr)                                          # record expression

        # print() ###
        # print('expr: ', end='') ###
        # pprint.pprint(expr) ###

        # convert end of f-string to str.format literal
        end = lineno[tokens[-1].end[0]] + tokens[-1].end[1]
        source[end:end+1] = '.format(%s)%s' % (', '.join(expr), source[end])

        # for each token, convert expression literals and brace '{}' escape sequences
        for token, entries in reversed(entryl):     # using reversed to keep offset in leading context
            token_start = lineno[token.start[0]] + token.start[1]   # actual offset at start of token
            token_end = lineno[token.end[0]] + token.end[1]         # actual offset at end of token
            if entries:     # for f-string expressions, replace with empty string ('')
                for entry in reversed(entries):
                    start = token_start + entry.start
                    end = token_start + entry.stop
                    source[start:end] = ''
            else:           # for escape sequences, double braces
                source[token_start:token_end] = re.sub(r'([{}])', r'\1\1', source[token_start:token_end])

            # strip leading f-string literals ('[fF]')
            string = source[token_start:token_start+3]
            if re.match(r'^(rf|fr|f)', string, re.IGNORECASE) is not None:
                source[token_start:token_start+3] = re.sub(r'[fF]', r'', string, count=1)

    # return modified context
    return str(source)
963409
def something(TokenClass):
    """"""
    def callback(lexer, match, context):
        text = match.group()
        if not text:
            return
        yield match.start(), TokenClass, text
        context.pos = match.end()
    return callback
860200
def lex(filename):
    """"""
    with io.open(filename, mode='r', encoding='utf-8') as f:
        it = _lex_file_object(f)
        it = _balance_braces(it, filename)
        for token, line, quoted in it:
            yield (token, line, quoted)
905105
def parse(self, text):
        
        tokens = self.lex(text)
        parser = Parser(tokens)
        return parser.parse()
1286382
def parse_string(string, options = {}, &block)
      parse_io(StringIO.new(string), options, &block)
    end
615178
def lex(input)
      @input = input

      return enum_for(:lex, input) unless block_given?

      if debug
        logger.info "lex: tokens   = #{@dsl.lex_tokens}"
        logger.info "lex: states   = #{@dsl.state_info}"
        logger.info "lex: ignore   = #{@dsl.state_ignore}"
        logger.info "lex: error    = #{@dsl.state_error}"
      end

      stream_tokens(input) do |token|
        yield token
      end
    end