Process one text `t` with tokenizer `tok`.
555732
def process_text(self, t:str, tok:BaseTokenizer) -> List[str]:
        ""
        for rule in self.pre_rules: t = rule(t)
        toks = tok.tokenizer(t)
        for rule in self.post_rules: toks = rule(toks)
        return toks
394342
def tokenize(cls, text, mode='c'):
        
        if mode == 'c':
            return [ch for ch in text]
        else:
            return [w for w in text.split()]
1237049
def crude_tokenizer(line):
    """"""
    tokens = []
    buffer = ''
    for c in line.strip():
        if c == ' ' or c in string.punctuation:
            if buffer:
                tokens.append(buffer)
                buffer = ''
        else:
            buffer += c
    if buffer: tokens.append(buffer)
    return tokens
940881
function tokenize(text) {
    var tokens = [];
    if (text && text.length > 0) {
        var token = '';
        for (var i = 0; i < text.length; i++) {
            var chr = text[i];
            if (breakingChars.indexOf(chr) >= 0) {
                if (token.length > 0) {
                    tokens.push(token);
                }
                token = '';
            }
            else {
                token += chr;
            }
        }
        if (token.length > 0) {
            tokens.push(token);
        }
    }
    return tokens;
}
1185615
def operator(self, text):
        """"""
        cls = self.OPERATORS[text]
        self.push_token(cls(text, self.lineno, self.offset))
46386
function tokenizer(input) {
  const topLevelTokens = getTopLevelTokens();

  const length = input.length;

  const output = [];

  let cursor = 0;
  let lastBreak = 0;

  while (cursor < length) {
    const slice = input.slice(cursor);
    const found = matchPattern(topLevelTokens, slice, false);

    if (found && input[cursor - 1] === '\\') {
      const text = input.slice(lastBreak, cursor - 1);
      if (text) {
        output.push(new Text(text));
      }

      const escapedText = found[1][0];
      output.push(new Text(escapedText));

      cursor += escapedText.length;
      lastBreak = cursor;
    } else if (found) {
      const [Tok, matches] = found;

      const text = input.slice(lastBreak, cursor);
      if (text) {
        output.push(new Text(text));
      }
      output.push(new Tok(...matches));

      cursor += matches[0].length;
      lastBreak = cursor;
    } else {
      cursor += 1;
    }
  }
  const text = input.slice(lastBreak, cursor);
  if (text) {
    output.push(new Text(text));
  }

  // if there are more closes than opens
  // intelligently remove extra ones
  return removeExtraCloses(output);
}
57185
def tokenizer(text):
    
    for entry in text.split('$$$$\n'):
        if entry.rstrip():
            lines_stream = deque(entry.split('\n'))
        else:
            continue

        # yield from _molfile(stream=lines_stream)
        for token in _molfile(stream=lines_stream):
            yield token

        if len(lines_stream):
            # yield from _sdfile(stream=lines_stream)
            for token in _sdfile(stream=lines_stream):
                yield token

    yield EndOfFile()
1237661
def tokenize(self, text):
        """"""
        text = self._clean_text(text)
        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        text = self._tokenize_chinese_chars(text)
        orig_tokens = whitespace_tokenize(text)
        split_tokens = []
        for token in orig_tokens:
            if self.do_lower_case and token not in self.never_split:
                token = token.lower()
                token = self._run_strip_accents(token)
            split_tokens.extend(self._run_split_on_punc(token))

        output_tokens = whitespace_tokenize(" ".join(split_tokens))
        return output_tokens
352587
public String senTokenize(String text){
		String ret = text;
		
		if (isTokenization){
			ret = PennTokenizer.tokenize(text);
		}
		
		return ret.trim();
	}
437414
def process_tokens(self, tokens):
        
        """
        for (tok_type, token, (start_row, _), _, _) in tokens:
            if tok_type == tokenize.STRING:
                self._check_string(token, start_row)
1020651
def transform_source(text):
    '''
    '''
    toks = tokenize.generate_tokens(StringIO(text).readline)
    result = []
    replacing_keyword = False
    for toktype, tokvalue, _, _, _ in toks:
        if toktype == tokenize.NAME and tokvalue == 'switch':
            result.extend([
                (tokenize.NAME, 'for'),
                (tokenize.NAME, '__case'),
                (tokenize.NAME, 'in'),
                (tokenize.NAME, '_Switch'),
                (tokenize.OP, '(')
            ])
            replacing_keyword = True
        elif toktype == tokenize.NAME and (tokvalue == 'case' or tokvalue == 'default'):
            result.extend([
                (tokenize.NAME, 'if'),
                (tokenize.NAME, '__case'),
                (tokenize.OP, '(')
            ])
            replacing_keyword = True
        elif replacing_keyword and tokvalue == ':':
            result.extend([
                (tokenize.OP, ')'),
                (tokenize.OP, ':')
            ])
            replacing_keyword = False
        else:
            result.append((toktype, tokvalue))
    return tokenize.untokenize(result)
125825
def trips_process_text():
    """"""
    if request.method == 'OPTIONS':
        return {}
    response = request.body.read().decode('utf-8')
    body = json.loads(response)
    text = body.get('text')
    tp = trips.process_text(text)
    return _stmts_from_proc(tp)
538870
def tokenize_command(text):
    
    if text.peek() == '\\':
        c = text.forward(1)
        tokens = set(string.punctuation + string.whitespace) - {'*'}
        while text.hasNext() and (c == '\\' or text.peek() not in tokens) and c not in MATH_TOKENS:
            c += text.forward(1)
        return c
1354571
def tokenize(string):
    """"""
    for match in TOKENS_REGEX.finditer(string):
        yield Token(match.lastgroup, match.group().strip(), match.span())
1156752
def handle(self,
               t_input: inference.TranslatorInput,
               t_output: inference.TranslatorOutput,
               t_walltime: float = 0.):
        
        print_attention_text(t_output.attention_matrix,
                             t_input.tokens,
                             t_output.tokens,
                             self.threshold)
39368
def handle_generic(tokens, options)
      t = Chronic.time_class.parse(options[:text])
      Span.new(t, t + 1)
    end
399299
def tokenize(self, text: str, model: object = None):
        
        sentences = re.split(self.pattern, text)
        return sentences