convert to a bag of tokens
974806
final protected BagOfSourcedTokens asBagOfSourcedTokens(SourcedStringWrapper w) 
    {
        if (w instanceof BagOfSourcedTokens) return (BagOfSourcedTokens)w;
        else {
            SourcedToken[] toks = tokenizer.sourcedTokenize(w.unwrap(), w.getSource());
            return new BagOfSourcedTokens(w.unwrap(), toks);
        }
    }
952102
def tokens(cls, tokens):
        
        return cls(Lnk.TOKENS, tuple(map(int, tokens)))
1363737
def _tokenize(self, comps):
        """"""
        ps = []
        for comp in comps:
            ps.extend([c.strip(' ,') for c in re.split(r'\s+(?=[^{}]*(?:\{|$))', comp)])
        return [p for p in ps if p]
263120
def _tokenize_latex(self, exp):
        
        tokens = []
        prevexp = ""
        while exp:
            t, exp = self._get_next_token(exp)
            if t.strip() != "":
                tokens.append(t)
            if prevexp == exp:
                break
            prevexp = exp
        return tokens
620300
def _prepare_reserved_tokens(reserved_tokens):
  """"""
  reserved_tokens = [tf.compat.as_text(tok) for tok in reserved_tokens or []]
  dups = _find_duplicates(reserved_tokens)
  if dups:
    raise ValueError("Duplicates found in tokens: %s" % dups)
  reserved_tokens_re = _make_reserved_tokens_re(reserved_tokens)
  return reserved_tokens, reserved_tokens_re
496079
def tokenize(self, token_list):
    """"""
    return [
        self._vocab_dict.get(token, self._vocab_dict[self.UNK])
        for token in token_list
    ]
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens
1340700
def split_tokens(self):
        """"""
        if not self._split_tokens_calculated:
            # split into items (whitespace split)
            self._split_tokens = self._line_str.split()
            self._split_tokens_calculated = True

        return self._split_tokens
551205
function token() {
  var len = arguments[0] === undefined ? 20 : arguments[0];
  var encoding = arguments[1] === undefined ? 'hex' : arguments[1];

  return (0, _mzCrypto.randomBytes)(len).then(function (buf) {
    return buf.toString(encoding);
  });
}
157699
def _segment_with_tokens(text, tokens):
        """"""
        list_form = []
        text_ptr = 0
        for token in tokens:
            inter_token_string = []
            while not text[text_ptr:].startswith(token):
                inter_token_string.append(text[text_ptr])
                text_ptr += 1
                if text_ptr >= len(text):
                    raise ValueError("Tokenization produced tokens that do not belong in string!")
            text_ptr += len(token)
            if inter_token_string:
                list_form.append(''.join(inter_token_string))
            list_form.append(token)
        if text_ptr < len(text):
            list_form.append(text[text_ptr:])
        return list_form
876465
def completer_tokenize(cls, value, min_length=3):
        ''''''
        tokens = list(itertools.chain(*[
            [m for m in n.split("'") if len(m) > min_length]
            for n in value.split(' ')
        ]))
        return list(set([value] + tokens + [' '.join(tokens)]))
668618
def make_oie_string(tokens: List[Token], tags: List[str]) -> str:
    
    frame = []
    chunk = []
    words = [token.text for token in tokens]

    for (token, tag) in zip(words, tags):
        if tag.startswith("I-"):
            chunk.append(token)
        else:
            if chunk:
                frame.append("[" + " ".join(chunk) + "]")
                chunk = []

            if tag.startswith("B-"):
                chunk.append(tag[2:] + ": " + token)
            elif tag == "O":
                frame.append(token)

    if chunk:
        frame.append("[" + " ".join(chunk) + "]")

    return " ".join(frame)
1280705
def tokenize(expr):
    
    tokens = []
    escape = False
    cur_token = ''

    for c in expr:
        if escape == True:
            cur_token += c
            escape = False
        else:
            if c == '\\':
                # Next char will be escaped
                escape = True
                continue
            elif c == '[':
                # Next token is of type index (list)
                if len(cur_token) > 0:
                    tokens.append(cur_token)
                    cur_token = ''
            elif c == ']':
                # End of index token. Next token defaults to a key (dict)
                if len(cur_token) > 0:
                    tokens.append(int(cur_token))
                    cur_token = ''
            elif c == '.':
                # End of key token. Next token defaults to a key (dict)
                if len(cur_token) > 0:
                    tokens.append(cur_token)
                    cur_token = ''
            else:
                # Append char to token name
                cur_token += c
    if len(cur_token) > 0:
        tokens.append(cur_token)

    return tokens
226491
def parse_tokens(data)
      tokens = {}
      data.split("\n").each do |t|
        tokens.merge!({ t.split('=').first.downcase.to_sym => t.split('=').last })
      end
      return tokens
    end
532194
def convert(document):
    """"""
    raw_tokens = []
    curpos = 0
    text_spans = []
    all_labels = []
    sent_spans = []
    word_texts = []
    for sentence in document:
        startpos = curpos
        for idx, (text, label) in enumerate(sentence):
            raw_tokens.append(text)
            word_texts.append(text)
            all_labels.append(label)
            text_spans.append((curpos, curpos+len(text)))
            curpos += len(text)
            if idx < len(sentence) - 1:
                raw_tokens.append(' ')
                curpos += 1
        sent_spans.append((startpos, curpos))
        raw_tokens.append('\n')
        curpos += 1
    return {
        TEXT: ''.join(raw_tokens),
        WORDS: [{TEXT: text, START: start, END: end, LABEL: label} for text, (start, end), label in zip(word_texts, text_spans, all_labels)],
        SENTENCES: [{START: start, END:end} for start, end in sent_spans]
    }
1319450
def decode(self, subtokens):
    """"""
    if isinstance(subtokens, np.ndarray):
      # Note that list(subtokens) converts subtokens to a python list, but the
      # items remain as np.int32. This converts both the array and its items.
      subtokens = subtokens.tolist()

    if not subtokens:
      return ""

    assert isinstance(subtokens, list) and isinstance(subtokens[0], int), (
        "Subtokens argument passed into decode() must be a list of integers.")

    return _unicode_to_native(
        _join_tokens_to_string(self._subtoken_ids_to_tokens(subtokens)))
478019
def _tokenize(self, text, token_class=None):
        
        token_class = token_class or Token
        tokens = {}

        for i, match in enumerate(self.regex.finditer(text)):
            value = match.group(0)

            try:
                token = tokens[value]
            except KeyError:
                type = match.lastgroup
                token = token_class(value, type=type)
                tokens[value] = token

            yield token