Iterate through all tokens in the input string.
197891
def _iter_tokens(self):
        """"""
        reobj, actions, nextstates = self._rules[self.states[-1]]
        mobj = reobj.match(self.string, self.pos)
        while mobj is not None:
            text = mobj.group(0)
            idx = mobj.lastindex - 1
            nextstate = nextstates[idx]

            # Take action
            actions[idx](self, text)
            while self.tokens:
                yield self.pop_token()
            if nextstate and nextstate != self.states[-1]:
                self.states[-1] = nextstate

            # Update position variables
            self.pos = mobj.end()
            lines = text.split('\n')
            nlines = len(lines) - 1
            if nlines == 0:
                self.offset = self.offset + len(lines[0])
            else:
                self.lineno = self.lineno + nlines
                self.offset = 1 + len(lines[-1])

            reobj, actions, nextstates = self._rules[self.states[-1]]
            mobj = reobj.match(self.string, self.pos)

        if self.pos != len(self.string):
            msg = "unexpected character"
            text = self.string[self.pos]
            raise RunError(msg, self.lineno, self.offset, text)

        yield EndToken("", self.lineno, self.offset)
625519
def tag(self, tokens):
        """"""
        if not self._loaded_model:
            self.load(self.model)
        tags = [None] * len(tokens)
        norm = self._normalize(tokens)
        length = len(norm)
        # A set of allowed indexes for matches to start or end at
        delims = [0] + [i for span in [m.span() for m in self.delimiters.finditer(norm)] for i in span] + [length]
        # Token indices
        token_at_index = []
        for i, t in enumerate(tokens):
            token_at_index.extend([i] * (len(self.lexicon[t].normalized) + 1))
        start_i = 0
        end_i = 1
        matches = {}
        next_start = end_i
        # TODO: This could be a little more efficient by skipping indexes forward to next delim points.
        while True:
            current = norm[start_i:end_i]
            if self._dawg.has_keys_with_prefix(current):
                # print('%s:%s:%s' % (start_i, end_i, current))
                # If the current span is in the dawg, and isn't followed by an alphanumeric character
                if current in self._dawg and start_i in delims and end_i in delims:
                    # print(current)
                    # Subsequent longer matches with same start_i will overwrite values in matches dict
                    matches[start_i] = (start_i, end_i, current)
                    # We can skip forward to after this match next time we increment start_i
                    next_start = end_i
                # Increment end_i provided we aren't already at the end of the input
                if end_i < length:
                    end_i += 1
                    continue
            # Increment start_i provided we aren't already at the end of the input
            start_i = next_start
            if start_i >= length - 1:
                break
            end_i = start_i + 1
            next_start = end_i
        # Apply matches as tags to the relevant tokens
        for start_i, end_i, current in matches.values():
            start_token = token_at_index[start_i]
            end_token = token_at_index[end_i]
            # Possible for match to start in 'I' token from prev match. Merge matches by not overwriting to 'B'.
            if not tags[start_token] == 'I-%s' % self.entity:
                tags[start_token] = 'B-%s' % self.entity
            tags[start_token+1:end_token+1] = ['I-%s' % self.entity] * (end_token - start_token)
        tokentags = list(zip(tokens, tags))
        return tokentags
1372767
def default_parse(self, string):
        
        results = list()
        while(string):
            token = self.longest(string)
            results.append(token)
            string = string[len(token):]
        return results
167694
def build_regex(self, tokens):
        """"""
        patterns = []
        for token in tokens:
            patterns.append(token.pattern_start)
            if token.pattern_end:
                patterns.append(token.pattern_end)
        return re.compile('|'.join(patterns), re.DOTALL)
1169048
public function tokenize($input) : iterable
    {
        if (!is_string($input)) {
            throw new core\exceptions\InvalidArgumentType($input, ['string']);
        }

        // Change line breaks, tabs etc. into whitespaces.
        $length = strlen($input);
        $cursor = 0;
        $i      = 0;
        $tokens = new formats\tokens\Argv;

        while ($cursor < $length) {
            if (preg_match('/\s+/A', $input, $match, null, $cursor)) {
            } elseif (preg_match('/([^="\'\s]+?)(=?)('.static::PATTERN_QUOTED.'+)/A', $input, $match, null, $cursor)) {
                $tokens->set($i++, $match[1].$match[2].stripcslashes(str_replace(['"\'', '\'"', '\'\'', '""'], '', substr($match[3], 1, -1))));
            } elseif (preg_match('/'.static::PATTERN_QUOTED.'/A', $input, $match, null, $cursor)) {
                $tokens->set($i++, stripcslashes(substr($match[0], 1, -1)));
            } elseif (preg_match('/'.static::PATTERN_BARE.'/A', $input, $match, null, $cursor)) {
                $tokens->set($i++, stripcslashes($match[1]));
            } else {
                throw new \RuntimeException('Failed to parse string near "... '.substr($input, $cursor, 10).' ..."');
            }

            $cursor += strlen($match[0]);
        }

        return $tokens;
    }
1245764
def parse(self, text):
        """"""
        text = self.preprocess(text)
        token_stack = []
        last_pos = 0

        # Iterate through all matched tokens
        for match in self.regex.finditer(text):
            # Find which token has been matched by regex
            token, match_type, group = self.get_matched_token(match)

            # Get params from stack of tokens
            params = self.get_params(token_stack)

            # Should we skip interpreting tokens?
            skip = token_stack[-1].skip if token_stack else False

            # Check for end token first
            if match_type == MatchType.end:
                if not skip or token_stack[-1] == token:
                    removed = self.remove_token(token_stack, token)
                    if removed:
                        skip = False
                    else:
                        skip = True

            if not skip:
                # Append text preceding matched token
                start_pos = match.start(group)
                if start_pos > last_pos:
                    yield Segment(self.postprocess(text[last_pos:start_pos]), **params)

                # Actions specific for start token or single token
                if match_type == MatchType.start:
                    token_stack.append(token)
                elif match_type == MatchType.single:
                    single_params = params.copy()
                    single_params.update(token.params)
                    single_text = token.text if token.text is not None else match.group(group)
                    yield Segment(single_text, token=token, match=match, **single_params)

                # Move last position pointer to the end of matched token
                last_pos = match.end(group)

        # Append anything that's left
        if last_pos < len(text):
            params = self.get_params(token_stack)
            yield Segment(self.postprocess(text[last_pos:]), **params)
334656
public static List<Token> tokenize(CharSequence input, boolean textTrim) {
        if (input == null) {
            throw new IllegalArgumentException("input must not be null");
        }

        int inputLen = input.length();
        if (inputLen == 0) {
            List<Token> tokens = new ArrayList<>(1);
            tokens.add(new Token(""));
            return tokens;
        }

        int status = AT_TEXT;
        int tokenStartOffset = 0; // start position of token in the stringBuffer
        char symbol = Token.PARAMETER_SYMBOL; // PARAMETER_SYMBOL or ATTRIBUTE_SYMBOL
        char c;

        StringBuilder nameBuf = new StringBuilder();
        StringBuilder valueBuf = new StringBuilder();
        StringBuilder textBuf = new StringBuilder();

        List<Token> tokens = new ArrayList<>();

        for (int i = 0; i < inputLen; i++) {
            c = input.charAt(i);

            switch (status) {
            case AT_TEXT:
                textBuf.append(c);
                if (Token.isTokenSymbol(c)) {
                    symbol = c;
                    status = AT_TOKEN_SYMBOL;
                    // abc$ --> tokenStartOffset: 3
                    tokenStartOffset = textBuf.length() - 1;
                }
                break;
            case AT_TOKEN_SYMBOL:
                textBuf.append(c);
                if (c == Token.START_BRACKET) {
                    status = AT_TOKEN_NAME;
                } else {
                    if (Token.isTokenSymbol(c)) {
                        symbol = c;
                        status = AT_TOKEN_SYMBOL;
                        // abc$ --> tokenStartOffset: 3
                        tokenStartOffset = textBuf.length() - 1;
                    } else {
                        status = AT_TEXT;
                    }
                }
                break;
            case AT_TOKEN_NAME:
            case AT_TOKEN_VALUE:
                textBuf.append(c);
                if (status == AT_TOKEN_NAME) {
                    if (c == Token.VALUE_SEPARATOR) {
                        status = AT_TOKEN_VALUE;
                        break;
                    }
                }
                if (c == Token.END_BRACKET) {
                    if (nameBuf.length() > 0 || valueBuf.length() > 0) {
                        // save previous non-token string
                        if (tokenStartOffset > 0) {
                            String text = trimBuffer(textBuf, tokenStartOffset, textTrim);
                            Token token = new Token(text);
                            tokens.add(token);
                        }

                        // save token name and default value
                        Token token = createToken(symbol, nameBuf, valueBuf);
                        tokens.add(token);

                        status = AT_TEXT;
                        textBuf.setLength(0);
                        break;
                    }

                    status = AT_TEXT;
                    break;
                }
                if (status == AT_TOKEN_NAME) {
                    if (nameBuf.length() > MAX_TOKEN_NAME_LENGTH) {
                        status = AT_TEXT;
                        nameBuf.setLength(0);
                    }
                    nameBuf.append(c);
                } else {
                    valueBuf.append(c);
                }
                break;
            }
        }

        if (textBuf.length() > 0) {
            String text = trimBuffer(textBuf, textBuf.length(), textTrim);
            Token token = new Token(text);
            tokens.add(token);
        }

        return tokens;
    }
825940
def _tokenize(self, stream):
        

        self._tokens = []
        self._reset_token()
        self._state = self.ST_TOKEN

        for chunk in iter(lambda: stream.read(8192), ''):
            for char in chunk:
                if char in self.NEWLINES:
                    self._process_newline(char)

                else:
                    state = self._state

                    if state == self.ST_STRING:
                        self._process_string(char)

                    elif state == self.ST_TOKEN:
                        self._process_tokens(char)
891917
def all_substrings(s):
    '''  '''
    join = ''.join
    for i in range(1, len(s) + 1):
        for sub in window(s, i):
            yield join(sub)
766321
def token_offsets(tokens):
    """"""
    end_offset = 0
    previous_end_row = 0
    previous_end_column = 0
    for t in tokens:
        token_type = t[0]
        token_string = t[1]
        (start_row, start_column) = t[2]
        (end_row, end_column) = t[3]

        # Account for the whitespace between tokens.
        end_offset += start_column
        if previous_end_row == start_row:
            end_offset -= previous_end_column

        # Record the start offset of the token.
        start_offset = end_offset

        # Account for the length of the token itself.
        end_offset += len(token_string)

        yield (token_type,
               token_string,
               start_offset,
               end_offset)

        previous_end_row = end_row
        previous_end_column = end_column
150778
def parse_tokens(self, tokens, debug=False):
        """"""
        # XXX Move the prefix computation into a wrapper around tokenize.
        p = parse.Parser(self.grammar, self.convert)
        p.setup()
        lineno = 1
        column = 0
        type = value = start = end = line_text = None
        prefix = u""
        for quintuple in tokens:
            type, value, start, end, line_text = quintuple
            if start != (lineno, column):
                assert (lineno, column) <= start, ((lineno, column), start)
                s_lineno, s_column = start
                if lineno < s_lineno:
                    prefix += "\n" * (s_lineno - lineno)
                    lineno = s_lineno
                    column = 0
                if column < s_column:
                    prefix += line_text[column:s_column]
                    column = s_column
            if type in (tokenize.COMMENT, tokenize.NL):
                prefix += value
                lineno, column = end
                if value.endswith("\n"):
                    lineno += 1
                    column = 0
                continue
            if type == token.OP:
                type = grammar.opmap[value]
            if debug:
                self.logger.debug("%s %r (prefix=%r)",
                                  token.tok_name[type], value, prefix)
            if p.addtoken(type, value, (prefix, start)):
                if debug:
                    self.logger.debug("Stop.")
                break
            prefix = ""
            lineno, column = end
            if value.endswith("\n"):
                lineno += 1
                column = 0
        else:
            # We never broke out -- EOF is too soon (how can this happen???)
            raise parse.ParseError("incomplete input",
                                   type, value, (prefix, start))
        return p.rootnode
675652
def tokenize(self, string):
        """"""

        if self.language == 'akkadian':
            tokens = tokenize_akkadian_words(string)
        elif self.language == 'arabic':
            tokens = tokenize_arabic_words(string)
        elif self.language == 'french':
            tokens = tokenize_french_words(string)
        elif self.language == 'greek':
            tokens = tokenize_greek_words(string)
        elif self.language == 'latin':
            tokens = tokenize_latin_words(string)
        elif self.language == 'old_norse':
            tokens = tokenize_old_norse_words(string)
        elif self.language == 'middle_english':
            tokens = tokenize_middle_english_words(string)
        elif self.language == 'middle_high_german':
            tokens = tokenize_middle_high_german_words(string)
        else:
            tokens = nltk_tokenize_words(string)

        return tokens
449716
def iter_finds(regex_obj, s):
    """"""
    if isinstance(regex_obj, str):
        for m in re.finditer(regex_obj, s):
            yield m.group()
    else:
        for m in regex_obj.finditer(s):
            yield m.group()
1337142
def iter_slices(string, slice_length):
    """"""
    pos = 0
    if slice_length is None or slice_length <= 0:
        slice_length = len(string)
    while pos < len(string):
        yield string[pos:pos + slice_length]
        pos += slice_length
282741
def keyword(tokens, expected):
    """"""
    try:
        token = next(iter(tokens))
    except StopIteration:
        return

    if token and token.name == "symbol" and token.value.lower() == expected:
        return TokenMatch(None, token.value, (token,))
909844
def remove_all(self, token):
        

        out = self.string.replace(" ", token)  # replace tokens
        while out.find(token + token) >= 0:  # while there are tokens
            out = out.replace(token + token, token)
        return out
1076911
def run_string(self, string, metadata=None):
        """"""
        token = Token(string, metadata)
        return [str(tkn) for tkn in self.run([token])]