Tokenizes documents in the corpus by word
188482
def words(self: object, fileids: str):
        
        for sent in self.sents(fileids):
            for token in word_tokenize(sent):
                yield token
709145
def macronize_text(self, text):
        
        macronized_words = [entry[2] for entry in self.macronize_tags(text)]
        return " ".join(macronized_words)
143301
def tokenize(self, data):
        '''
        

        '''
        super().tokenize(data)
        token_tuple_zip = self.n_gram.generate_tuple_zip(self.token, self.n)
        token_list = []
        self.token = ["".join(list(token_tuple)) for token_tuple in token_tuple_zip]
305735
def tokenize(s):
    
    return [m.group(m.lastindex) for m in _tdl_re.finditer(s)]
987583
def tokenize_paragraphs(self):
        """"""
        tok = self.__paragraph_tokenizer
        spans = tok.span_tokenize(self.text)
        dicts = []
        for start, end in spans:
            dicts.append({'start': start, 'end': end})
        self[PARAGRAPHS] = dicts
        return self
390902
def _tokenize(self, text):
        """"""
        text = self._clean_text(text)

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        text = self._tokenize_chinese_chars(text)
        orig_tokens = self._whitespace_tokenize(text)
        split_tokens = []
        for token in orig_tokens:
            if self.lower:
                token = token.lower()
                token = self._run_strip_accents(token)
            split_tokens.extend(self._run_split_on_punc(token))

        output_tokens = self._whitespace_tokenize(' '.join(split_tokens))
        return output_tokens
783490
def pos_tokenize(self: object, fileids: str):
        
        for para in self.paras(fileids):
            yield [
                self.pos_tagger(word_tokenize(sent))
                for sent in sent_tokenize(para)
            ]
532194
def convert(document):
    """"""
    raw_tokens = []
    curpos = 0
    text_spans = []
    all_labels = []
    sent_spans = []
    word_texts = []
    for sentence in document:
        startpos = curpos
        for idx, (text, label) in enumerate(sentence):
            raw_tokens.append(text)
            word_texts.append(text)
            all_labels.append(label)
            text_spans.append((curpos, curpos+len(text)))
            curpos += len(text)
            if idx < len(sentence) - 1:
                raw_tokens.append(' ')
                curpos += 1
        sent_spans.append((startpos, curpos))
        raw_tokens.append('\n')
        curpos += 1
    return {
        TEXT: ''.join(raw_tokens),
        WORDS: [{TEXT: text, START: start, END: end, LABEL: label} for text, (start, end), label in zip(word_texts, text_spans, all_labels)],
        SENTENCES: [{START: start, END:end} for start, end in sent_spans]
    }
376928
def tokenize_words(string):
    
    string = six.text_type(string)
    return re.findall(WORD_TOKENIZATION_RULES, string)
1356764
def iter_tokens(cls, blob):
    
    readline_func = io.StringIO(blob.decode('utf-8')).readline
    return tokenize.generate_tokens(readline_func)
394342
def tokenize(cls, text, mode='c'):
        
        if mode == 'c':
            return [ch for ch in text]
        else:
            return [w for w in text.split()]
34604
def generate_corpus(self, text):
        
        if isinstance(text, str):
            sentences = self.sentence_split(text)
        else:
            sentences = []
            for line in text:
                sentences += self.sentence_split(line)
        passing = filter(self.test_sentence_input, sentences)
        runs = map(self.word_split, passing)
        return runs
1067408
@SneakyThrows
   public Corpus process(@NonNull Corpus documents) {
      timer.start();

      Broker.Builder<Document> builder = Broker.<Document>builder()
                                            .addProducer(new IterableProducer<>(documents))
                                            .bufferSize(queueSize);

      Corpus corpus = documents;
      if (returnCorpus && corpus.getDataSetType() == DatasetType.OffHeap) {
         Resource tempFile = Resources.temporaryDirectory();
         tempFile.deleteOnExit();
         try (MultiFileWriter writer = new MultiFileWriter(tempFile, "part-", Config.get("files.partition")
                                                                                    .asIntegerValue(numberOfThreads))) {
            builder.addConsumer(new AnnotateConsumer(annotationTypes, onComplete, documentsProcessed, wordsProcessed, writer),
                                numberOfThreads)
                   .build()
                   .run();
         }
         corpus = Corpus.builder().offHeap().source(CorpusFormats.JSON_OPL, tempFile).build();
      } else {
         builder.addConsumer(new AnnotateConsumer(annotationTypes, onComplete, documentsProcessed,wordsProcessed, null),
                             numberOfThreads)
                .build()
                .run();
      }

      timer.stop();
      totalTime += timer.elapsed(TimeUnit.NANOSECONDS);
      timer.reset();

      return corpus;
   }
57185
def tokenizer(text):
    
    for entry in text.split('$$$$\n'):
        if entry.rstrip():
            lines_stream = deque(entry.split('\n'))
        else:
            continue

        # yield from _molfile(stream=lines_stream)
        for token in _molfile(stream=lines_stream):
            yield token

        if len(lines_stream):
            # yield from _sdfile(stream=lines_stream)
            for token in _sdfile(stream=lines_stream):
                yield token

    yield EndOfFile()
1181840
def _tokenize(cls, sentence):
        
        while True:
            match = cls._regex_tag.search(sentence)
            if not match:
                yield from cls._split(sentence)
                return
            chunk = sentence[:match.start()]
            yield from cls._split(chunk)
            tag = match.group(0)
            yield tag
            sentence = sentence[(len(chunk) + len(tag)):]
811278
def tag(words: List[str], corpus: str = "pud") -> List[Tuple[str, str]]:
    
    if not words:
        return []

    if corpus == "orchid":
        tagger = _ORCHID_TAGGER
        i = 0
        while i < len(words):
            if words[i] == " ":
                words[i] = "<space>"
            elif words[i] == "+":
                words[i] = "<plus>"
            elif words[i] == "-":
                words[i] = "<minus>"
            elif words[i] == "=":
                words[i] = "<equal>"
            elif words[i] == ",":
                words[i] = "<comma>"
            elif words[i] == "$":
                words[i] = "<dollar>"
            elif words[i] == ".":
                words[i] = "<full_stop>"
            elif words[i] == "(":
                words[i] = "<left_parenthesis>"
            elif words[i] == ")":
                words[i] = "<right_parenthesis>"
            elif words[i] == '"':
                words[i] = "<quotation>"
            elif words[i] == "@":
                words[i] = "<at_mark>"
            elif words[i] == "&":
                words[i] = "<ampersand>"
            elif words[i] == "{":
                words[i] = "<left_curly_bracket>"
            elif words[i] == "^":
                words[i] = "<circumflex_accent>"
            elif words[i] == "?":
                words[i] = "<question_mark>"
            elif words[i] == "<":
                words[i] = "<less_than>"
            elif words[i] == ">":
                words[i] = "<greater_than>"
            elif words[i] == "=":
                words[i] = "<equal>"
            elif words[i] == "!":
                words[i] = "<exclamation>"
            elif words[i] == "’":
                words[i] = "<apostrophe>"
            elif words[i] == ":":
                words[i] = "<colon>"
            elif words[i] == "*":
                words[i] = "<asterisk>"
            elif words[i] == ";":
                words[i] = "<semi_colon>"
            elif words[i] == "/":
                words[i] = "<slash>"
            i += 1
        t2 = tagger.tag(words)
        t = []
        i = 0
        while i < len(t2):
            word = t2[i][0]
            tag = t2[i][1]
            if word == "<space>":
                word = " "
            elif word == "<plus>":
                word = "+"
            elif word == "<minus>":
                word = "-"
            elif word == "<equal>":
                word = "="
            elif word == "<comma>":
                word = ","
            elif word == "<dollar>":
                word = "$"
            elif word == "<full_stop>":
                word = "."
            elif word == "<left_parenthesis>":
                word = "("
            elif word == "<right_parenthesis>":
                word = ")"
            elif word == "<quotation>":
                word = '"'
            elif word == "<at_mark>":
                word = "@"
            elif word == "<ampersand>":
                word = "&"
            elif word == "<left_curly_bracket>":
                word = "{"
            elif word == "<circumflex_accent>":
                word = "^"
            elif word == "<question_mark>":
                word = "?"
            elif word == "<less_than>":
                word = "<"
            elif word == "<greater_than>":
                word = ">"
            elif word == "<equal>":
                word = "="
            elif word == "<exclamation>":
                word = "!"
            elif word == "<apostrophe>":
                word = "’"
            elif word == "<colon>":
                word = ":"
            elif word == "<asterisk>":
                word = "*"
            elif word == "<semi_colon>":
                word = ";"
            elif word == "<slash>":
                word = "/"
            t.append((word, tag))
            i += 1
    else:  # default, use "pud" as a corpus
        tagger = _PUD_TAGGER
        t = tagger.tag(words)

    return t
1304807
function(str, lex) {
  if (lex) {
    w.plugin({
      words: lex
    });
  }
  let doc = buildText(str, w);
  doc.tagger();
  return doc;
}