Save the tokenizer vocabulary to a directory or file.
198795
def save_vocabulary(self, vocab_path):
        """"""
        index = 0
        if os.path.isdir(vocab_path):
            vocab_file = os.path.join(vocab_path, VOCAB_NAME)
        with open(vocab_file, "w", encoding="utf-8") as writer:
            for token, token_index in sorted(self.vocab.items(), key=lambda kv: kv[1]):
                if index != token_index:
                    logger.warning("Saving vocabulary to {}: vocabulary indices are not consecutive."
                                   " Please check that the vocabulary is not corrupted!".format(vocab_file))
                    index = token_index
                writer.write(token + u'\n')
                index += 1
        return vocab_file
1358946
def tokenize_words(self, text):
        """"""
        return [
            self.strip_punctuation(word) for word in text.split(' ')
            if self.strip_punctuation(word)
        ]
1219493
def tokenize(s):
    

    # Crude tokenization
    s = s.replace('+', ' PLUS ').replace('-', ' MINUS ') \
        .replace('/', ' DIVIDE ').replace('*', ' MULTIPLY ')
    words = [w for w in s.split(' ') if w]
    return words
21798
def add_token_to_namespace(self, token: str, namespace: str = 'tokens') -> int:
        
        if not isinstance(token, str):
            raise ValueError("Vocabulary tokens must be strings, or saving and loading will break."
                             "  Got %s (with type %s)" % (repr(token), type(token)))
        if token not in self._token_to_index[namespace]:
            index = len(self._token_to_index[namespace])
            self._token_to_index[namespace][token] = index
            self._index_to_token[namespace][index] = token
            return index
        else:
            return self._token_to_index[namespace][token]
642380
def _load_vocab_file(vocab_file, reserved_tokens=None):
  """"""
  if reserved_tokens is None:
    reserved_tokens = RESERVED_TOKENS

  subtoken_list = []
  with tf.gfile.Open(vocab_file, mode="r") as f:
    for line in f:
      subtoken = _native_to_unicode(line.strip())
      subtoken = subtoken[1:-1]  # Remove surrounding single-quotes
      if subtoken in reserved_tokens:
        continue
      subtoken_list.append(_native_to_unicode(subtoken))
  return reserved_tokens + subtoken_list
1199831
def _save_vocab_file(vocab_file, subtoken_list):
  """"""
  with tf.gfile.Open(vocab_file, mode="w") as f:
    for subtoken in subtoken_list:
      f.write("'%s'\n" % _unicode_to_native(subtoken))
48449
private function saveToken()
	{
		$fd = fopen($this->cacheFolder."/token.txt", 'w');
		fwrite($fd, $this->token);
		fclose($fd);
	}
57185
def tokenizer(text):
    
    for entry in text.split('$$$$\n'):
        if entry.rstrip():
            lines_stream = deque(entry.split('\n'))
        else:
            continue

        # yield from _molfile(stream=lines_stream)
        for token in _molfile(stream=lines_stream):
            yield token

        if len(lines_stream):
            # yield from _sdfile(stream=lines_stream)
            for token in _sdfile(stream=lines_stream):
                yield token

    yield EndOfFile()
965248
def _init_vocab(self, token_generator, add_reserved_tokens=True):
    """"""

    self._id_to_token = {}
    non_reserved_start_index = 0

    if add_reserved_tokens:
      self._id_to_token.update(enumerate(RESERVED_TOKENS))
      non_reserved_start_index = len(RESERVED_TOKENS)

    self._id_to_token.update(
        enumerate(token_generator, start=non_reserved_start_index))

    # _token_to_id is the reverse of _id_to_token
    self._token_to_id = dict((v, k)
                             for k, v in six.iteritems(self._id_to_token))
588062
def save(file_name, content):
    """"""
    with open(file_name, "w", encoding="utf-8") as output_file:
        output_file.write(content)
        return output_file.name
733002
def tokenize(contents):
    """"""
    tokens = _scan_for_tokens(contents)
    tokens = _compress_tokens(tokens)
    tokens = [token for token in tokens if token.type != TokenType.Whitespace]
    return tokens
652877
def save(f, arr, vocab):
    
    f.write(('%d %d' % (arr.shape[0], arr.shape[1])).encode('utf-8'))
    for word, idx in vocab:
        _write_line(f, arr[idx], word)
149714
def save_token(self):
        
        if self.token is None:
            raise ValueError('You have to set the "token" first.')

        try:
            if not self.token_path.parent.exists():
                self.token_path.parent.mkdir(parents=True)
        except Exception as e:
            log.error('Token could not be saved: {}'.format(str(e)))
            return False

        with self.token_path.open('w') as token_file:
            # 'indent = True' will make the file human readable
            self.serializer.dump(self.token, token_file, indent=True)

        return True
1141125
def prepare_to_generate(self, data_dir, tmp_dir):
    """"""
    self.get_or_create_vocab(data_dir, tmp_dir)
    self.train_text_filepaths(tmp_dir)
    self.dev_text_filepaths(tmp_dir)
390902
def _tokenize(self, text):
        """"""
        text = self._clean_text(text)

        # This was added on November 1st, 2018 for the multilingual and Chinese
        # models. This is also applied to the English models now, but it doesn't
        # matter since the English models were not trained on any Chinese data
        # and generally don't have any Chinese data in them (there are Chinese
        # characters in the vocabulary because Wikipedia does have some Chinese
        # words in the English Wikipedia.).
        text = self._tokenize_chinese_chars(text)
        orig_tokens = self._whitespace_tokenize(text)
        split_tokens = []
        for token in orig_tokens:
            if self.lower:
                token = token.lower()
                token = self._run_strip_accents(token)
            split_tokens.extend(self._run_split_on_punc(token))

        output_tokens = self._whitespace_tokenize(' '.join(split_tokens))
        return output_tokens
592977
def save(self, filename='classifier.dump'):
        
        ofile = open(filename,'w+')
        pickle.dump(self.classifier, ofile)
        ofile.close()
784830
public function save()
    {
        $keys = array_keys($this->langstrings);
        sort($keys);

        $maxlen       = 0;
        $langPrefixes = array();
        foreach ($keys as $key) {
            $tokens = explode('.', $key);
            if (preg_match('/tl_layout\.[a-z]+\.css\./', $key)) {
                $tokens = array($tokens[0], $tokens[1] . '.' . $tokens[2], $tokens[3]);
            }
            $langPrefix = '$GLOBALS[\'TL_LANG\']';
            foreach ($tokens as $token) {
                $langPrefix .= '[\'' . $token . '\']';
            }

            $langPrefixes[$key] = $langPrefix;

            if (strlen($langPrefix) > $maxlen) {
                $maxlen = strlen($langPrefix);
            }
        }

        $buffer = $this->getHead() . PHP_EOL . PHP_EOL;
        foreach ($keys as $key) {
            $prefix  = $langPrefixes[$key];
            $buffer .= sprintf(
                '%s = %s;' . PHP_EOL,
                $prefix . str_repeat(' ', ($maxlen - strlen($prefix))),
                var_export($this->getValue($key), true)
            );
        }
        $buffer  .= PHP_EOL;
        $resource = fopen($this->filename, 'wb');
        fputs($resource, $buffer);

        fclose($resource);
    }