Initialize vocabulary with tokens from token_generator.
965248
def _init_vocab(self, token_generator, add_reserved_tokens=True):
    """"""

    self._id_to_token = {}
    non_reserved_start_index = 0

    if add_reserved_tokens:
      self._id_to_token.update(enumerate(RESERVED_TOKENS))
      non_reserved_start_index = len(RESERVED_TOKENS)

    self._id_to_token.update(
        enumerate(token_generator, start=non_reserved_start_index))

    # _token_to_id is the reverse of _id_to_token
    self._token_to_id = dict((v, k)
                             for k, v in six.iteritems(self._id_to_token))
776243
def _add_pret_words(self, pret_embeddings):
        
        words_in_train_data = set(self._id2word)
        pret_embeddings = gluonnlp.embedding.create(pret_embeddings[0], source=pret_embeddings[1])

        for idx, token in enumerate(pret_embeddings.idx_to_token):
            if token not in words_in_train_data:
                self._id2word.append(token)
965209
def load_vocab(vocab_file):
    """"""
    vocab = collections.OrderedDict()
    index = 0
    with io.open(vocab_file, 'r') as reader:
        while True:
            token = reader.readline()
            if not token:
                break
            token = token.strip()
            vocab[token] = index
            index += 1
    return vocab
1352544
def gen(self, init_state=None):
        
        state = init_state or (BEGIN,) * self.state_size
        while True:
            next_word = self.move(state)
            if next_word == END: break
            yield next_word
            state = tuple(state[1:]) + (next_word,)
939722
def from_tokens(cls, path:PathOrStr, trn_tok:Collection[Collection[str]], trn_lbls:Collection[Union[int,float]],
                 val_tok:Collection[Collection[str]], val_lbls:Collection[Union[int,float]], vocab:Vocab=None,
                 tst_tok:Collection[Collection[str]]=None, classes:Collection[Any]=None, max_vocab:int=60000, min_freq:int=3,
                 **kwargs) -> DataBunch:
        ""
        processor = NumericalizeProcessor(vocab=vocab, max_vocab=max_vocab, min_freq=min_freq)
        src = ItemLists(path, TextList(trn_tok, path=path, processor=processor),
                        TextList(val_tok, path=path, processor=processor))
        src = src.label_for_lm() if cls==TextLMDataBunch else src.label_from_lists(trn_lbls, val_lbls, classes=classes)
        if tst_tok is not None: src.add_test(TextList(tst_tok, path=path))
        return src.databunch(**kwargs)
818522
def _init_vocab_from_file(self, filename):
    
    with tf.gfile.Open(filename) as f:
      tokens = [token.strip() for token in f.readlines()]

    def token_gen():
      for token in tokens:
        yield token

    self._init_vocab(token_gen(), add_reserved_tokens=False)
285860
def get_docs(self, vocab):
        """"""
        for string in self.strings:
            vocab[string]
        orth_col = self.attrs.index(ORTH)
        for tokens, spaces in zip(self.tokens, self.spaces):
            words = [vocab.strings[orth] for orth in tokens[:, orth_col]]
            doc = Doc(vocab, words=words, spaces=spaces)
            doc = doc.from_array(self.attrs, tokens)
            yield doc
1119872
def _token_to_subwords(self, token):
    """"""
    subwords = []

    start = 0
    while start < len(token):
      subword = None
      for end in range(
          min(len(token), start + self._max_subword_len), start, -1):
        candidate = token[start:end]
        if (candidate in self._subword_to_id or
            candidate == _UNDERSCORE_REPLACEMENT):
          subword = candidate
          subwords.append(subword)
          start = end
          break
      # No subword match found. Consume a single (unicode) character.
      if subword is None:
        subwords.append(token[start])
        start += 1

    return subwords
443201
def build_input_data(sentences, labels, vocabulary):
    """"""
    x = np.array([[vocabulary[word] for word in sentence] for sentence in sentences])
    y = np.array(labels)
    return [x, y]
741807
def tokens(self):
    
    seq = self.word_tokenizer.transform(Sequence(self.raw))
    return WordList(seq.tokens(), parent=self, language=self.language.code)
314848
def word_tokenize(text, stopwords=_stopwords, ngrams=None, min_length=0, ignore_numeric=True):
    
    if ngrams is None:
        ngrams = 1

    text = re.sub(re.compile('\'s'), '', text)  # Simple heuristic
    text = re.sub(_re_punctuation, '', text)

    matched_tokens = re.findall(_re_token, text.lower())
    for tokens in get_ngrams(matched_tokens, ngrams):
        for i in range(len(tokens)):
            tokens[i] = tokens[i].strip(punctuation)

            if len(tokens[i]) < min_length or tokens[i] in stopwords:
                break
            if ignore_numeric and isnumeric(tokens[i]):
                break
        else:
            yield tuple(tokens)
944448
function init(app, context, swagger) {
        logger.log('debug', '%s|adding|routes|context=%s', meta.module, context, meta);
        if (swagger) {
            describeModels(swagger);
            swagger.addPost({
                'spec': {
                    "path" : context + format + '/lex/phrase', 
                    "notes" : "To use only the lexer to tokenize a phrase. The output of the lexer can be pass directly to the tokenizer.",
                    "method": "POST",
                    "summary" : "Split a phrase into tokens.",
                    "nickname" : "lexer",
                    "responseClass" : "Tokens",
                    "params" : [
                        swagger.params.post("phrase", "A phrase to be tokenized", '{"phrase":"Hello world"}'),
                    ],
                    "errorResponses" : [
                        swagger.errors.notFound('phrase'),
                        swagger.errors.invalid('phrase')
                    ]
                },
                'action': function(req, res) {
                    if (!req.body)
                        throw swagger.error(400, 'Missing body of the request');
                    if (!req.body.phrase)
                        throw swagger.error(400, 'Missing the "phrase" in the body of the request');
                    pos.rest.lexPhrase(req, res);
                }
            }).addPost({
                'spec': {
                    "path" : context + format + '/tag/words',
                    "notes" : "To use the tagger on a list of tokens.",
                    "method": "POST",
                    "summary" : "Tag each tokenized word.",
                    "nickname" : "word-tagger",
                    "responseClass" : "Tags",
                    "params" : [
                        swagger.params.post("words", "A list of tokenized words to be tagged", '{"words":["hello","world"]}'),
                    ],
                    "errorResponses" : [
                        swagger.errors.notFound('words'),
                        swagger.errors.invalid('words')
                    ]
                },
                'action': function(req, res) {
                    if (!req.body)
                        throw swagger.error(400, 'Missing body of the request');
                    if (!req.body.words)
                        throw swagger.error(400, 'Missing the "words" array in the body of the request');
                    pos.rest.tagWords(req, res);
                }
            }).addPost({
                'spec': {
                    "path" : context + format + '/tag/phrase',
                    "notes" : "Use the lexer on the phrase and, tag the resulting tokens.",
                    "method": "POST",
                    "summary" : "Tokenize a phrase and then tag each token.",
                    "nickname" : "tagger",
                    "responseClass" : "Tags",
                    "params" : [
                        swagger.params.post("phrase", "A phrase to be tokenized and tagged", '{"phrase":"Hello world"}'),
                    ],
                    "errorResponses" : [
                        swagger.errors.notFound('phrase'),
                        swagger.errors.invalid('phrase')
                    ]
                },
                'action': function(req, res) {
                    if (!req.body)
                        throw swagger.error(400, 'Missing body of the request');
                    if (!req.body.phrase)
                        throw swagger.error(400, 'Missing the "phrase" in the body of the request');
                    pos.rest.tagPhrase(req, res);
                }
            });
        } else {
            router.add(app, context + '/pos.json/lex/phrase', 'POST', pos.rest.lexPhrase);
            router.add(app, context + '/pos.json/tag/words',  'POST', pos.rest.tagWords);
            router.add(app, context + '/pos.json/tag/phrase', 'POST', pos.rest.tagPhrase);
        }
    }
229339
def prepare_tokens(ctx, input, annotations, tout, lout):
    """"""
    click.echo('chemdataextractor.chemdner.prepare_tokens')
    # Collect the annotations into a dict
    anndict = defaultdict(list)
    for line in annotations:
        pmid, ta, start, end, text, category = line.strip().split('\t')
        anndict[(pmid, ta)].append((int(start), int(end), text))
    # Process the corpus
    for line in input:
        pmid, title, abstract = line.strip().split(u'\t')
        for t, section, anns in [(Title(title), 'T', anndict.get((pmid, u'T'), [])), (Paragraph(abstract), u'A', anndict.get((pmid, u'A'), []))]:
            # Write our tokens with POS and IOB tags
            tagged = _prep_tags(t, anns)
            for i, sentence in enumerate(tagged):
                tout.write(u' '.join([u'/'.join([token, tag, label]) for token, tag, label in sentence]))
                lout.write(u' '.join([u'/'.join([token, label]) for token, tag, label in sentence]))
                tout.write(u'\n')
                lout.write(u'\n')
            tout.write(u'\n')
            lout.write(u'\n')
771994
def generate_tokens(self, text):
        """"""
        if text != self.last_text:
            string_io = io.StringIO(text)
            self.last_tokens = list(
                tokenize.generate_tokens(string_io.readline)
            )
            self.last_text = text
        return self.last_tokens
145155
def init_word_db(cls, name, text):
        """"""
        # Prep the words
        text = text.replace('\n', ' ').replace('\r', ' ')
        words = [w.strip() for w in text.split(' ') if w.strip()]

        assert len(words) > 2, \
                'Database text sources must contain 3 or more words.'

        # Build the database
        freqs = {}
        for i in range(len(words) - 2):

            # Create a triplet from the current word
            w1 = words[i]
            w2 = words[i + 1]
            w3 = words[i + 2]

            # Add the triplet to the database
            key = (w1, w2)
            if key in freqs:
                freqs[key].append(w3)
            else:
                freqs[key] = [w3]

        # Store the database so it can be used
        cls._dbs[name] = {
            'freqs': freqs,
            'words': words,
            'word_count': len(words) - 2
            }
1063303
def generate_initialization_vector(words)
    srand(Time.now.to_i)
    vector = ""
    words.times {
      vector << [rand(ULONG)].pack('N')
    }
    return(vector)
  end
667228
def load_initial(self, streams):
        """"""
        d = {}
        for stream in streams:
            s = io.load(stream)
            if 'BLOCK' not in s:
                raise ValueError("No BLOCK found")
            d.update(s['BLOCK'])
        d = {'BLOCK': d}
        C = io.wc_lha2dict(d)
        sm = io.sm_lha2dict(d)
        C.update(sm)
        C = definitions.symmetrize(C)
        self.C_in = C