Converts a sequence of ids in wordpiece tokens using the vocab.
970208
def convert_ids_to_tokens(self, ids):
        """"""
        tokens = []
        for i in ids:
            tokens.append(self.ids_to_tokens[i])
        return tokens
835351
def synset(self, id, pos=ADJECTIVE):
        
        id = str(id).zfill(8)
        if not id.startswith(("n-", "v-", "a-", "r-")):
            if pos == NOUN:
                id = "n-" + id
            if pos == VERB:
                id = "v-" + id
            if pos == ADJECTIVE:
                id = "a-" + id
            if pos == ADVERB:
                id = "r-" + id
        if dict.__len__(self) == 0:
            self.load()
        try:
            return tuple(self._synsets[id])[:2]
        except KeyError: # Some WordNet id's are not zero padded.
            return tuple(self._synsets.get(re.sub(r"-0+", "-", id), (0.0, 0.0))[:2])
223991
def token_generator(self, texts, **kwargs):
        
        for text_idx, text in enumerate(texts):
            if self.lower:
                text = text.lower()
            for char in text:
                yield text_idx, char
238620
def tokens2ids(tokens: Iterable[str], vocab: Dict[str, int]) -> List[int]:
    
    return [vocab.get(w, vocab[C.UNK_SYMBOL]) for w in tokens]
303159
def vocab_freq(docs, limit=1e6, verbose=1, tokenizer=generate_tokens):
    
    total = Counter()
    try:
        limit = min(limit, docs.count())
        docs = docs.iterator()
    except:
        pass
    for i, doc in enumerate(docs):
        try:
            doc = doc.values()
        except AttributeError:
            if not isinstance(doc, basestring):
                doc = ' '.join([stringify(v) for v in doc])
            else:
                doc = stringify(doc)
        if i >= limit:
            break
        c = Counter(tokenizer(doc, strip=True, nonwords=False))
        if verbose and (verbose < 1e-3 or not i % int(limit * verbose)):
            print('{}: {} ... {}'.format(i, c.keys()[:3], c.keys()[-3:] if len(c.keys()) > 6 else ''))
        total += c
    return total
877182
def convert_to_sequences(dataset, vocab):
    
    start = time.time()
    dataset_vocab = map(lambda x: (x, vocab), dataset)
    with mp.Pool() as pool:
        # Each sample is processed in an asynchronous manner.
        output = pool.map(get_sequence, dataset_vocab)
    end = time.time()
    logging.info('Done! Sequence conversion Time={:.2f}s, #Sentences={}'
                 .format(end - start, len(dataset)))
    return output
430200
def _iterate_subsequences(self, tokens):
        
        for start_idx in xrange(len(tokens)):
            for end_idx in xrange(start_idx + 1, len(tokens) + 1):
                yield ' '.join(tokens[start_idx:end_idx]), start_idx
1267428
def tokenize(self, sentence_str):
        '''
        
        '''
        mt = MeCab.Tagger("-Owakati")
        wordlist = mt.parse(sentence_str)
        token_list = wordlist.rstrip(" \n").split(" ")
        return token_list
1119872
def _token_to_subwords(self, token):
    """"""
    subwords = []

    start = 0
    while start < len(token):
      subword = None
      for end in range(
          min(len(token), start + self._max_subword_len), start, -1):
        candidate = token[start:end]
        if (candidate in self._subword_to_id or
            candidate == _UNDERSCORE_REPLACEMENT):
          subword = candidate
          subwords.append(subword)
          start = end
          break
      # No subword match found. Consume a single (unicode) character.
      if subword is None:
        subwords.append(token[start])
        start += 1

    return subwords
25308
def tokens(self):
        """"""
        spans = self.word_tokenizer.span_tokenize(self.text)
        toks = [Token(
            text=self.text[span[0]:span[1]],
            start=span[0] + self.start,
            end=span[1] + self.start,
            lexicon=self.lexicon
        ) for span in spans]
        return toks
18236
def from_tokens(cls, text, toks):
        
        sentence_pb = CoreNLP_pb2.Sentence()
        sentence_pb.characterOffsetBegin = 0
        sentence_pb.characterOffsetEnd = len(text)
        sentence_pb.sentenceIndex = 0
        sentence_pb.tokenOffsetBegin = 0
        sentence_pb.tokenOffsetEnd = len(toks)

        # Track progress in sentence and tokens.
        char_idx = 0
        tok_idx = 0

        buf = ""
        token_pb = None
        while char_idx < len(text):
            tok = toks[tok_idx]
            # Scan to the beginning of the token.
            if text[char_idx] != tok[0]:
                buf += text[char_idx]
                char_idx += 1
            # Aha! we have found the token. Assert that they match.
            else:
                assert text[char_idx:char_idx+len(tok)] == tok, "text did not match a token"
                # Create a new Token from this text.
                if token_pb: token_pb.after = buf
                token_pb = sentence_pb.token.add()
                token_pb.before = buf

                token_pb.beginChar = char_idx
                token_pb.endChar = char_idx + len(tok)
                # TODO(chaganty): potentially handle -LRB-?
                token_pb.value = tok
                token_pb.word = tok
                token_pb.originalText = text[char_idx:char_idx+len(tok)]

                buf = ""
                char_idx += len(tok)
                tok_idx += 1
        if token_pb: token_pb.after = buf
        assert tok_idx == len(toks), "text does not match all tokens"
        return AnnotatedSentence.from_pb(sentence_pb)
25336
def _tokens_to_subtoken_ids(self, tokens):
    
    ret = []
    for token in tokens:
      ret.extend(self._token_to_subtoken_ids(token))
    return ret
80936
def _trim_and_decode(ids, subtokenizer):
  """"""
  try:
    index = list(ids).index(tokenizer.EOS_ID)
    return subtokenizer.decode(ids[:index])
  except ValueError:  # No EOS found in sequence
    return subtokenizer.decode(ids)
1219493
def tokenize(s):
    

    # Crude tokenization
    s = s.replace('+', ' PLUS ').replace('-', ' MINUS ') \
        .replace('/', ' DIVIDE ').replace('*', ' MULTIPLY ')
    words = [w for w in s.split(' ') if w]
    return words
1183690
def _subtoken_ids_to_tokens(self, subtokens):
    
    concatenated = "".join(
        [self._subtoken_id_to_subtoken_string(s) for s in subtokens])
    split = concatenated.split("_")
    ret = []
    for t in split:
      if t:
        unescaped = _unescape_token(t + "_")
        if unescaped:
          ret.append(unescaped)
    return ret
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens
661874
def tokenize(self, text):
        '''
        
        '''
        start = -1
        tokens = []
        for i, character in enumerate(text):
            if character == ' ' or character == '\t':
                if start >= 0:
                    word = text[start:i]
                    tokens.append({
                        'word': word,
                        'original_text': word,
                        'char_begin': start,
                        'char_end': i})
                    start = -1
            else:
                if start < 0:
                    start = i
        if start >= 0:
            tokens.append({
                'word': text[start:len(text)],
                'original_text': text[start:len(text)],
                'char_begin': start,
                'char_end': len(text)
            })
        return tokens