Tokenizes the string.

        :param s: the string to tokenize
        :type s: str
        :return: the iterator
        :rtype: TokenIterator
197048
def tokenize(self, s):
        
        javabridge.call(self.jobject, "tokenize", "(Ljava/lang/String;)V", s)
        return TokenIterator(self)
478019
def _tokenize(self, text, token_class=None):
        
        token_class = token_class or Token
        tokens = {}

        for i, match in enumerate(self.regex.finditer(text)):
            value = match.group(0)

            try:
                token = tokens[value]
            except KeyError:
                type = match.lastgroup
                token = token_class(value, type=type)
                tokens[value] = token

            yield token
341388
final boolean tokenIs(String s)
  {
    return (m_token != null) ? (m_token.equals(s)) : (s == null);
  }
542112
def serialiseString(self, s):
        
        if type(s) is unicode:
            s = self.context.getBytesForString(s)

        self.serialiseBytes(s)
693912
def tokenize(self, string):
        '''
        
        '''

        new_lexer = ply.lex.lex(module=self, debug=self.debug, errorlog=logger)
        new_lexer.latest_newline = 0
        new_lexer.string_value = None
        new_lexer.input(string)

        while True:
            t = new_lexer.token()
            if t is None: break
            t.col = t.lexpos - new_lexer.latest_newline
            yield t

        if new_lexer.string_value is not None:
            raise JsonPathLexerError('Unexpected EOF in string literal or identifier')
952111
final public StringWrapper prepare(String s) {
		BagOfTokens bag = new BagOfTokens(s, tokenizer.tokenize(s));
		double totalWeight = bag.getTotalWeight();
		for (Iterator i=bag.tokenIterator(); i.hasNext(); ) {
			Token tok = (Token)i.next();
			double freq = bag.getWeight(tok);
			bag.setWeight( tok, smoothedProbability(tok, freq, totalWeight) );
		}
		return bag;
	}
399808
def process_tokens(self, tokens):
        
        for tok_type, token, (start_row, start_col), _, _ in tokens:
            if tok_type == tokenize.STRING:
                # 'token' is the whole un-parsed token; we can look at the start
                # of it to see whether it's a raw or unicode string etc.
                self._process_string_token(token, start_row, start_col)
899146
def tokenize(self, s):
    """"""
    s = tf.compat.as_text(s)

    if self.reserved_tokens:
      # First split out the reserved tokens
      substrs = self._reserved_tokens_re.split(s)
    else:
      substrs = [s]

    toks = []
    for substr in substrs:
      if substr in self.reserved_tokens:
        toks.append(substr)
      else:
        toks.extend(self._alphanum_re.split(substr))

    # Filter out empty strings
    toks = [t for t in toks if t]
    return toks
825940
def _tokenize(self, stream):
        

        self._tokens = []
        self._reset_token()
        self._state = self.ST_TOKEN

        for chunk in iter(lambda: stream.read(8192), ''):
            for char in chunk:
                if char in self.NEWLINES:
                    self._process_newline(char)

                else:
                    state = self._state

                    if state == self.ST_STRING:
                        self._process_string(char)

                    elif state == self.ST_TOKEN:
                        self._process_tokens(char)
1070446
def tokenize_sents(string):
    
    string = six.text_type(string)

    spans = []
    for match in re.finditer('[^\s]+', string):
        spans.append(match)
    spans_count = len(spans)

    rez = []
    off = 0

    for i in range(spans_count):
        tok = string[spans[i].start():spans[i].end()]
        if i == spans_count - 1:
            rez.append(string[off:spans[i].end()])
        elif tok[-1] in ['.', '!', '?', '…', '»']:
            tok1 = tok[re.search('[.!?…»]', tok).start()-1]
            next_tok = string[spans[i + 1].start():spans[i + 1].end()]
            if (next_tok[0].isupper()
                and not tok1.isupper()
                and not (tok[-1] != '.'
                         or tok1[0] == '('
                         or tok in ABBRS)):
                rez.append(string[off:spans[i].end()])
                off = spans[i + 1].start()

    return rez
565660
public static String[] spansToStrings(Error[] spans, CharSequence s) {
    String[] tokens = new String[spans.length];

    for (int si = 0, sl = spans.length; si < sl; si++) {
      tokens[si] = spans[si].getCoveredText(s).toString();
    }

    return tokens;
  }
1321558
def span_tokenize(self, s):
        
        if self._tokenizer is None:
            self._tokenizer = load_model(self.model)
        # for debug in tokenizer.debug_decisions(s):
        #     log.debug(format_debug_decision(debug))
        return self._tokenizer.span_tokenize(s)
1161598
def encode(self, s):
        
        if isinstance(s, basestring) and self.__needs_encoding(s):
            for x in self.encodings:
                s = re.sub(x[0], x[1], s)
        return s
753616
def tokenize(self, string):
        ''''''
        pos = 0
        while pos < len(string):
            m = SKIP_RE.match(string, pos=pos)
            if m:
                skip_lines = m.group(0).split('\n')
                if len(skip_lines) > 1:
                    self.row += len(skip_lines) - 1
                    self.column = 1 + len(skip_lines[-1])
                else:
                    self.column += len(skip_lines[0])

                pos = m.end()
                continue

            for cls, type, regex in self.token_map:
                m = regex.match(string, pos=pos)
                if m:
                    yield cls(type, m.group(0),
                              self.filename, self.row, self.column)
                    self.column += len(m.group(0))
                    pos = m.end()
                    break
            else:
                raise ConfigParseError(
                    "Couldn't load config in %r row %d, column %d: %r" %
                    (self.filename, self.row, self.column,
                     string[pos:pos+20]))
1198380
def _parse_tokens(tokens):
    

    index = 0
    parsed_tokens = []

    num_tokens = len(tokens)
    while index < num_tokens:
        tok = Token(*tokens[index])

        assert tok.token_type != token.INDENT
        if tok.token_type == tokenize.NEWLINE:
            # There's only one newline and it's at the end.
            break

        if tok.token_string in '([{':
            (container, index) = _parse_container(tokens, index)
            if not container:
                return None
            parsed_tokens.append(container)
        else:
            parsed_tokens.append(Atom(tok))

        index += 1

    return parsed_tokens
1058906
def _get_token_type(self, char):
        
        if char in '()':
            return self.SPLIT, 0
        elif char == ',':
            return self.SPLIT, 1
        elif char in '<>':
            return self.IGNORE, 2
        elif char == '.':
            return self.JOIN, 3
        elif char.isdigit():
            return self.JOIN, 4
        elif char.isalpha():
            return self.JOIN, 5
        elif char == '^':
            return self.JOIN, 6
        elif char.isspace():
            return self.IGNORE, 7
        else:
            raise ValueError('TokeniserException: "{}" can not be tokenised'.format(char))
743699
@Nonnull
    public static String[] tokenize(@Nonnull String s, @CheckForNull String delimiter) {
        return QuotedStringTokenizer.tokenize(s,delimiter);
    }