A generator that generates samples that are encoded.

    Args:
      data_dir: data directory
      tmp_dir: temp directory
      dataset_split: dataset split

    Yields:
      A dict.
331461
def generate_encoded_samples(self, data_dir, tmp_dir, dataset_split):
    
    generator = self.generate_samples(data_dir, tmp_dir, dataset_split)
    encoder = self.get_or_create_vocab(data_dir, tmp_dir)
    label_encoder = self.get_labels_encoder(data_dir)
    for sample in generator:
      inputs = encoder.encode(sample["inputs"])
      inputs.append(text_encoder.EOS_ID)
      context = encoder.encode(sample["context"])
      context.append(text_encoder.EOS_ID)
      targets = label_encoder.encode(sample["targets"])
      sample["targets"] = targets
      yield {"inputs": inputs, "context": context, "targets": targets}
647105
def _generate_examples(self, data_file):
    """"""
    target_language = self.builder_config.target_language

    with tf.io.gfile.GFile(data_file) as f:
      for i, line in enumerate(f):
        line_parts = line.strip().split("\t")
        if len(line_parts) != 2:
          raise ValueError(("Wrong data format in line {}. The line '{}' does "
                            "not have exactly one delimiter.").format(i, line))
        source, target = line_parts[0].strip(), line_parts[1].strip()

        yield {"en": source, target_language: target}
272836
def batch_gen(data, batch_size):
    '''
    
    '''
    data = data or []
    for i in range(0, len(data), batch_size):
        yield data[i:i + batch_size]
1214940
def generate_samples(self):
        """"""
        filenames = glob_all(self.args.random_data_folder, '*.wav')
        shuffle(filenames)
        while True:
            for fn in filenames:
                for x, y in self.vectors_from_fn(fn):
                    yield x, y
120100
def temporary_dir():
    
    d = tempfile.mkdtemp()
    try:
        with cd(d):
            yield d
    finally:
        if os.path.exists(d):
            shutil.rmtree(d)
956405
def dataset(ctx, client, revision, datadir, format):
    """"""
    ctx.meta['renku.datasets.datadir'] = datadir

    if ctx.invoked_subcommand is not None:
        return

    if revision is None:
        datasets = client.datasets.values()
    else:
        datasets = client.datasets_from_commit(client.repo.commit(revision))

    DATASETS_FORMATS[format](client, datasets)
1139643
def data_files(self):
    
    tf_record_pattern = os.path.join(FLAGS.data_dir, '%s-*' % self.subset)
    data_files = tf.gfile.Glob(tf_record_pattern)
    if not data_files:
      print('No files found for dataset %s/%s at %s' % (self.name,
                                                        self.subset,
                                                        FLAGS.data_dir))

      self.download_message()
      exit(-1)
    return data_files
540787
def make_data_splits(label_type, train_rec_type="text_and_wordlist", max_samples=1000,
                     seed=0, tgt_dir=TGT_DIR):
    

    feat_dir = os.path.join(tgt_dir, "feat")
    test_prefix_fn=os.path.join(tgt_dir, "test_prefixes.txt")
    valid_prefix_fn=os.path.join(tgt_dir, "valid_prefixes.txt")
    with open(test_prefix_fn) as f:
        prefixes = f.readlines()
        test_prefixes = [("TEXT/" + prefix).strip() for prefix in prefixes]
    with open(valid_prefix_fn) as f:
        prefixes = f.readlines()
        valid_prefixes = [("TEXT/" + prefix).strip() for prefix in prefixes]

    label_dir = os.path.join(tgt_dir, "label")
    prefixes = get_story_prefixes(label_type, label_dir=label_dir)
    prefixes = list(set(prefixes) - set(valid_prefixes))
    prefixes = list(set(prefixes) - set(test_prefixes))
    prefixes = utils.filter_by_size(
        feat_dir, prefixes, "fbank", max_samples)

    if train_rec_type == "text":
        train_prefixes = prefixes
    else:
        wordlist_prefixes = [prefix for prefix in os.listdir(os.path.join(label_dir, "WORDLIST"))
                             if prefix.endswith("phonemes")]
        wordlist_prefixes = [os.path.splitext(os.path.join("WORDLIST", prefix))[0]
                             for prefix in wordlist_prefixes]
        wordlist_prefixes = utils.filter_by_size(
                feat_dir, wordlist_prefixes, "fbank", max_samples)
        if train_rec_type == "wordlist":
            prefixes = wordlist_prefixes
        elif train_rec_type == "text_and_wordlist":
            prefixes.extend(wordlist_prefixes)
        else:
            raise PersephoneException("train_rec_type='%s' not supported." % train_rec_type)
        train_prefixes = prefixes
    random.seed(seed)
    random.shuffle(train_prefixes)

    return train_prefixes, valid_prefixes, test_prefixes
985270
def _tempDirectories(self):
        
        def _dirs(path, levels):
            if levels > 0:
                for subPath in os.listdir(path):
                    for i in _dirs(os.path.join(path, subPath), levels-1):
                        yield i
            else:
                yield path
        for tempDir in _dirs(self.tempFilesDir, self.levels):
            yield tempDir
520227
def generate_data(self, data_dir, tmp_dir, task_id=-1):
    
    tf.logging.info("generate_data task_id=%s" % task_id)
    encoder = self.get_or_create_vocab(data_dir, tmp_dir)
    assert task_id >= 0 and task_id < self.num_generate_tasks
    if task_id < self.num_train_shards:
      out_file = self.training_filepaths(
          data_dir, self.num_train_shards, shuffled=False)[task_id]
    else:
      out_file = self.dev_filepaths(
          data_dir, self.num_dev_shards,
          shuffled=False)[task_id - self.num_train_shards]
    generator_utils.generate_files(
        self.example_generator(encoder, tmp_dir, task_id), [out_file])
    generator_utils.shuffle_dataset([out_file])
266125
def samples(self):
        """"""
        names = self.series.dimensions
        for values in zip(*(getattr(self.series, name) for name in names)):
            yield dict(zip(names, values))
257143
def _get_files_generator(src_dir, files_list):
    
    if files_list:
        for d in files_list:
            yield {
                'src': os.path.join(src_dir, d['src']),
                'dst': _get_dst_dir(d['dst']),
                'post_commands': d.get('post_commands', []),
            }
965691
def get_or_generate_vocabulary(data_dir,
                               tmp_dir,
                               data_prefix,
                               max_page_size_exp,
                               approx_vocab_size=32768,
                               strip=True):
  
  num_pages_for_vocab_generation = approx_vocab_size // 3
  vocab_file = vocab_filename(approx_vocab_size, strip)

  def my_generator(data_prefix):
    """Line generator for vocab."""
    count = 0
    for page in corpus_page_generator(
        all_corpus_files(data_prefix)[::-1], tmp_dir, max_page_size_exp):
      revisions = page["revisions"]
      if revisions:
        text = get_text(revisions[-1], strip=strip)
        yield text
        count += 1
        if count % 100 == 0:
          tf.logging.info("reading pages for vocab %d" % count)
        if count > num_pages_for_vocab_generation:
          break

  return generator_utils.get_or_generate_vocab_inner(data_dir, vocab_file,
                                                     approx_vocab_size,
                                                     my_generator(data_prefix))
654720
def _split_generators(self, dl_manager):
    """"""
    image_tar_file = os.path.join(dl_manager.manual_dir,
                                  self.builder_config.file_name)
    if not tf.io.gfile.exists(image_tar_file):
      # The current celebahq generation code depends on a concrete version of
      # pillow library and cannot be easily ported into tfds.
      msg = "You must download the dataset files manually and place them in: "
      msg += dl_manager.manual_dir
      msg += " as .tar files. See testing/test_data/fake_examples/celeb_a_hq "
      raise AssertionError(msg)
    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            num_shards=50,
            gen_kwargs={"archive": dl_manager.iter_archive(image_tar_file)},
        )
    ]
319172
def num_samples(self, dataset_split):
    
    return {
        problem.DatasetSplit.TRAIN: 1000000,
        problem.DatasetSplit.EVAL: 10000,
        problem.DatasetSplit.TEST: 10000
    }[dataset_split]
1129051
def directory_generator(dirname, trim=0):
    

    def gather(collect, dirname, fnames):
        for fname in fnames:
            df = join(dirname, fname)
            if not isdir(df):
                collect.append(df)

    collect = list()
    walk(dirname, gather, collect)
    for fname in collect:
        yield fname[trim:], file_chunk(fname)
700446
def snli_token_generator(tmp_dir, train, vocab_size):
  """"""
  _download_and_parse_dataset(tmp_dir, train)

  symbolizer_vocab = _get_or_generate_vocab(
      tmp_dir, 'vocab.subword_text_encoder', vocab_size)

  file_name = 'train' if train else 'dev'
  data_file = os.path.join(tmp_dir, file_name + '.txt')
  with tf.gfile.GFile(data_file, mode='r') as f:
    for line in f:
      sent1, sent2, label = line.strip().split('\t')
      sent1_enc = symbolizer_vocab.encode(sent1)
      sent2_enc = symbolizer_vocab.encode(sent2)

      inputs = sent1_enc + [_SEP] + sent2_enc + [_EOS]
      yield {
          'inputs': inputs,
          'targets': [_LABEL_TO_ID[label]],
      }