Saves source vocabularies (primary surface form vocabulary) and optional factor vocabularies to folder.

    :param source_vocabs: List of source vocabularies.
    :param folder: Destination folder.
425072
def save_source_vocabs(source_vocabs: List[Vocab], folder: str):
    
    for i, vocab in enumerate(source_vocabs):
        vocab_to_json(vocab, os.path.join(folder, C.VOCAB_SRC_NAME % i))
913327
def collect_vocab(qp_pairs):
    '''
    
    '''
    vocab = set()
    for qp_pair in qp_pairs:
        for word in qp_pair['question_tokens']:
            vocab.add(word['word'])
        for word in qp_pair['passage_tokens']:
            vocab.add(word['word'])
    return vocab
652069
def load_target_vocab(folder: str) -> Vocab:
    
    return vocab_from_json(os.path.join(folder, C.VOCAB_TRG_NAME % 0))
1324045
def read_vocab(args, column_name):
  
  vocab_path = os.path.join(args.analysis,
                            feature_transforms.VOCAB_ANALYSIS_FILE % column_name)

  if not file_io.file_exists(vocab_path):
    return []

  vocab, _ = feature_transforms.read_vocab_file(vocab_path)
  return vocab
764061
def save_to_files(self, directory: str) -> None:
        
        os.makedirs(directory, exist_ok=True)
        if os.listdir(directory):
            logging.warning("vocabulary serialization directory %s is not empty", directory)

        with codecs.open(os.path.join(directory, NAMESPACE_PADDING_FILE), 'w', 'utf-8') as namespace_file:
            for namespace_str in self._non_padded_namespaces:
                print(namespace_str, file=namespace_file)

        for namespace, mapping in self._index_to_token.items():
            # Each namespace gets written to its own file, in index order.
            with codecs.open(os.path.join(directory, namespace + '.txt'), 'w', 'utf-8') as token_file:
                num_tokens = len(mapping)
                start_index = 1 if mapping[0] == self._padding_token else 0
                for i in range(start_index, num_tokens):
                    print(mapping[i].replace('\n', '@@NEWLINE@@'), file=token_file)
378136
def load_common(self, wlist):
        """"""
        self.com_dict = os.path.join(self.base_dir, 'dicts', 'EN_vocab.txt')
        with open(self.com_dict) as words_file:
            data = words_file.read()
        self.common_words = set(data.splitlines())
        if wlist:
            new_words = ''
            for wl in wlist:
                with open(wl) as f:
                    new_words += f.read()
            if new_words:
                new_dict = set(data.splitlines())
                self.common_words.update(new_dict)
1264752
def src_vocab(self):
        
        if self._src_vocab is None:
            src_vocab_file_name, src_vocab_hash = \
                self._data_file[self._pair_key]['vocab' + '_' + self._src_lang]
            [src_vocab_path] = self._fetch_data_path([(src_vocab_file_name, src_vocab_hash)])
            with io.open(src_vocab_path, 'r', encoding='utf-8') as in_file:
                self._src_vocab = Vocab.from_json(in_file.read())
        return self._src_vocab
1199831
def _save_vocab_file(vocab_file, subtoken_list):
  """"""
  with tf.gfile.Open(vocab_file, mode="w") as f:
    for subtoken in subtoken_list:
      f.write("'%s'\n" % _unicode_to_native(subtoken))
1083426
def _build_vocab(self, sentences, markers=[]):
        
        from snorkel.learning.pytorch.rnn.utils import SymbolTable

        vocab = Counter()
        for sent in sentences:
            for w in sent:
                vocab[w] += 1
        word_dict = SymbolTable()
        list(map(word_dict.get, vocab))
        list(map(word_dict.get, markers))
        return word_dict
330621
def generate_data_with_shared_vocab(self, data_dir, tmp_dir, task_id=-1):
    """"""
    global_vocab_filename = os.path.join(data_dir, self.vocab_filename)
    if not tf.gfile.Exists(global_vocab_filename):
      raise ValueError(
          'Global vocabulary file: %s does not exist, '
          'please create one using build_vocab.py' % global_vocab_filename)
    # Before generating data, we copy the global vocabulary file to the children
    # locations. Although this is not the most disk efficient strategy, it
    # imposes the fewest changes to the text-to-text API.
    for p in self.problems:
      local_vocab_filename = os.path.join(data_dir, p.vocab_filename)
      if not tf.gfile.Exists(local_vocab_filename):
        tf.gfile.Copy(global_vocab_filename, local_vocab_filename)
      p.generate_data(data_dir, tmp_dir, task_id)
403406
def _get_or_generate_vocab(tmp_dir, vocab_filename, vocab_size):
  """"""
  vocab_filepath = os.path.join(tmp_dir, vocab_filename)
  print('Vocab file written to: ' + vocab_filepath)

  if tf.gfile.Exists(vocab_filepath):
    gs = text_encoder.SubwordTextEncoder(vocab_filepath)
    return gs
  example_file = os.path.join(tmp_dir, _EXAMPLES_FILE)
  gs = text_encoder.SubwordTextEncoder()
  token_counts = tokenizer.corpus_token_counts(
      example_file, corpus_max_lines=1000000)
  gs = gs.build_to_target_size(
      vocab_size, token_counts, min_val=1, max_val=1e3)
  gs.store_to_file(vocab_filepath)
  return gs
48109
def get_content_dict(vocabularies, content_vocab):
    
    # Raise exception if the vocabulary can't be found.
    if vocabularies.get(content_vocab, None) is None:
        raise UNTLFormException(
            'Could not retrieve content vocabulary "%s" for the form.'
            % (content_vocab)
        )
    else:
        # Return the sorted vocabulary.
        return vocabularies.get(content_vocab)
132347
def save_data_as(self, data, file_name, destination_folder=None):
        
        if not destination_folder:
            destination_folder = constants.Files.DOWNLOADS_FOLDER
        page_utils._save_data_as(data, destination_folder, file_name)
858566
public static void writeVocabCache(@NonNull VocabCache<VocabWord> vocabCache, @NonNull File file)
            throws IOException {
        try (FileOutputStream fos = new FileOutputStream(file)) {
            writeVocabCache(vocabCache, fos);
        }
    }
908252
@SuppressWarnings("unchecked") // method is safe, since all calls inside are using generic SequenceElement methods
    public VocabCache<T> buildMergedVocabulary(@NonNull WordVectors wordVectors, boolean fetchLabels) {
        return buildMergedVocabulary((VocabCache<T>) wordVectors.vocab(), fetchLabels);
    }
893294
def saveTopicPageDefinitionToFile(self, fname):
        
        open(fname, "w", encoding="utf-8").write(json.dumps(self.topicPage, indent = 4, sort_keys = True))
899782
def all(cls):
		'''

		'''
#		if vocab_id_or_name:
#			vocab = vocabulary.Vocabulary.get(vocab_id_or_name)
#			if vocab is None:
#				# The user specified an invalid vocab.
#				raise ckan.logic.NotFound("could not find vocabulary '%s'"
#						% vocab_id_or_name)
#		   query = meta.Session.query(Tag).filter(Tag.vocabulary_id==vocab.id)
#		else:
		query = meta.Session.query(SemanticTag)
		query = query.distinct().join(TagSemanticTag)
#		query = query.filter_by(state='active')
		return query