Splits of data to produce and number of output shards for each.
435581
def dataset_splits(self):
    """"""
    return [{
        "split": problem.DatasetSplit.TRAIN,
        "shards": _TRAIN_SHARDS,
    }, {
        "split": problem.DatasetSplit.EVAL,
        "shards": _DEV_SHARDS,
    }]
241626
def generate_data_for_problem(problem):
  """"""
  training_gen, dev_gen, test_gen = _SUPPORTED_PROBLEM_GENERATORS[problem]

  num_train_shards = FLAGS.num_shards or 10
  tf.logging.info("Generating training data for %s.", problem)
  train_output_files = generator_utils.train_data_filenames(
      problem + generator_utils.UNSHUFFLED_SUFFIX, FLAGS.data_dir,
      num_train_shards)
  generator_utils.generate_files(training_gen(), train_output_files,
                                 FLAGS.max_cases)
  num_dev_shards = int(num_train_shards * 0.1)
  tf.logging.info("Generating development data for %s.", problem)
  dev_output_files = generator_utils.dev_data_filenames(
      problem + generator_utils.UNSHUFFLED_SUFFIX, FLAGS.data_dir,
      num_dev_shards)
  generator_utils.generate_files(dev_gen(), dev_output_files)
  num_test_shards = int(num_train_shards * 0.1)
  test_output_files = []
  test_gen_data = test_gen()
  if test_gen_data is not None:
    tf.logging.info("Generating test data for %s.", problem)
    test_output_files = generator_utils.test_data_filenames(
        problem + generator_utils.UNSHUFFLED_SUFFIX, FLAGS.data_dir,
        num_test_shards)
    generator_utils.generate_files(test_gen_data, test_output_files)
  all_output_files = train_output_files + dev_output_files + test_output_files
  generator_utils.shuffle_dataset(all_output_files)
485846
def _prepare_output(partitions, verbose):
    """"""
    out = {}
    partitions_count = len(partitions)
    out['raw'] = {
        'offline_count': partitions_count,
    }

    if partitions_count == 0:
        out['message'] = 'No offline partitions.'
    else:
        out['message'] = "{count} offline partitions.".format(count=partitions_count)
        if verbose:
            lines = (
                '{}:{}'.format(topic, partition)
                for (topic, partition) in partitions
            )
            out['verbose'] = "Partitions:\n" + "\n".join(lines)
        else:
            cmdline = sys.argv[:]
            cmdline.insert(1, '-v')
            out['message'] += '\nTo see all offline partitions run: ' + ' '.join(cmdline)

    if verbose:
        out['raw']['partitions'] = [
            {'topic': topic, 'partition': partition}
            for (topic, partition) in partitions
        ]

    return out
511662
def iterchunks(data, chunksize):
    
    offt = 0
    while offt < len(data):
        yield data[offt:offt+chunksize]
        offt += chunksize
403628
def data_chunk(data, chunk, with_overlap=False):
    """"""
    assert isinstance(chunk, tuple)
    if len(chunk) == 2:
        i, j = chunk
    elif len(chunk) == 4:
        if with_overlap:
            i, j = chunk[:2]
        else:
            i, j = chunk[2:]
    else:
        raise ValueError("'chunk' should have 2 or 4 elements, "
                         "not {0:d}".format(len(chunk)))
    return data[i:j, ...]
604775
def results_aggregator(results):
    '''
    
    '''

    def _flatten_unique(list_of_lists):
        l_flat = sum(list_of_lists, [])
        l_unique = list(set(l_flat))
        return l_unique

    def _mean(numbers):
        return float(sum(numbers)) / max(len(numbers), 1)

    def _merge_response_times(response_times):
        flat_list = []
        for r_time in response_times:
            for key, value in r_time.items():
                flat_list.extend([int(float(key))] * value)
        hist, bins = histogram(flat_list)
        return {
            'histogram': hist.tolist(),
            'bins': bins.tolist(),
        }

    def _get_min(data, key):
        try:
            return min([r[key] for r in data if r[key] is not None])
        except ValueError:
            return 0

    def _get_max(data, key):
        try:
            return max([r[key] for r in data if r[key] is not None])
        except ValueError:
            return 0

    def _calculate_aws_lambda_cost(total_execution_time, memory_limit, invocation_count):
        dollar_cost_per_128mb_100ms = 0.000000208
        dollar_cost_per_invocation = 0.0000002
        memory_cost_multiplier = int(memory_limit) / 128.0
        time_in_100ms_lots = int(total_lambda_execution_time / 100.0)
        invocation_cost = invocation_count * 0.0000002
        execution_time_cost = time_in_100ms_lots * dollar_cost_per_128mb_100ms * memory_cost_multiplier
        return (invocation_cost + execution_time_cost)

    request_tasks = _flatten_unique([list(stat['requests'].keys()) for stat in results])
    failed_tasks = _flatten_unique([list(stat['failures'].keys()) for stat in results])
    total_lambda_execution_time = sum([(300000 - stat['remaining_time']) for stat in results])
    memory_limit = _get_max(results, 'memory_limit')

    agg_results = {
        'requests': {key: {} for key in request_tasks},
        'failures': {key: {} for key in failed_tasks},
        'num_requests': sum([stat['num_requests'] for stat in results]),
        'num_requests_fail': sum([stat['num_requests_fail'] for stat in results]),
        'total_lambda_execution_time': total_lambda_execution_time,
        'lambda_invocations': len(results),
        'approximate_cost': _calculate_aws_lambda_cost(total_lambda_execution_time, memory_limit, len(results))
    }

    for task in request_tasks:
        task_data_results = [stat['requests'][task] for stat in results if task in stat['requests']]

        for mean_stat in ['median_response_time', 'total_rps', 'avg_response_time']:
            agg_results['requests'][task][mean_stat] = _mean([data[mean_stat] for data in task_data_results])

        agg_results['requests'][task]['max_response_time'] = _get_max(task_data_results, 'max_response_time')
        agg_results['requests'][task]['min_response_time'] = _get_min(task_data_results, 'min_response_time')
        agg_results['requests'][task]['response_times'] = _merge_response_times([data['response_times'] for data in task_data_results if data['response_times'] is not None])
        agg_results['requests'][task]['total_rpm'] = agg_results['requests'][task]['total_rps'] * 60
        agg_results['requests'][task]['num_requests'] = sum([stat['requests'][task]['num_requests'] for stat in results if task in stat['requests']])

    for task in failed_tasks:
        task_data_results = [stat['failures'][task] for stat in results if task in stat['failures']]
        agg_results['failures'][task] = task_data_results[0]
        agg_results['failures'][task]['occurences'] = sum([stat['occurences'] for stat in task_data_results])

    return agg_results
1321641
def copy_data(from_client, from_project, from_logstore, from_time, to_time=None,
              to_client=None, to_project=None, to_logstore=None,
              shard_list=None,
              batch_size=None, compress=None, new_topic=None, new_source=None):
    
    to_client = to_client or from_client
    # increase the timeout to 2 min at least
    from_client.timeout = max(from_client.timeout, 120)
    to_client.timeout = max(to_client.timeout, 120)

    to_project = to_project or from_project
    to_logstore = to_logstore or from_logstore
    to_time = to_time or "end"

    cpu_count = multiprocessing.cpu_count() * 2
    shards = from_client.list_shards(from_project, from_logstore).get_shards_info()
    current_shards = [str(shard['shardID']) for shard in shards]
    target_shards = _parse_shard_list(shard_list, current_shards)
    worker_size = min(cpu_count, len(target_shards))

    result = dict()
    total_count = 0
    with ProcessPoolExecutor(max_workers=worker_size) as pool:
        futures = [pool.submit(copy_worker, from_client, from_project, from_logstore, shard,
                               from_time, to_time,
                               to_client, to_project, to_logstore,
                               batch_size=batch_size, compress=compress,
                               new_topic=new_topic, new_source=new_source)
                   for shard in target_shards]

        for future in as_completed(futures):
            partition, count = future.result()
            total_count += count
            if count:
                result[partition] = count

    return LogResponse({}, {"total_count": total_count, "shards": result})
413440
def _split_generators(self, dl_manager):
    
    path = dl_manager.download_and_extract(_DOWNLOAD_URL)
    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TEST,
            num_shards=1,
            gen_kwargs={'data_dir': os.path.join(path, _DIRNAME)})
    ]
757480
def _split_generators(self, dl_manager):
    """"""
    # Download images and annotations that come in separate archives.
    # Note, that the extension of archives is .tar.gz even though the actual
    # archives format is uncompressed tar.
    dl_paths = dl_manager.download_and_extract({
        "images": tfds.download.Resource(
            url=os.path.join(_BASE_URL, "102flowers.tgz"),
            extract_method=tfds.download.ExtractMethod.TAR),
        "labels": os.path.join(_BASE_URL, "imagelabels.mat"),
        "setid": os.path.join(_BASE_URL, "setid.mat"),
    })

    gen_kwargs = dict(
        images_dir_path=os.path.join(dl_paths["images"], "jpg"),
        labels_path=dl_paths["labels"],
        setid_path=dl_paths["setid"],
    )

    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            num_shards=1,
            gen_kwargs=dict(split_name="trnid", **gen_kwargs)),
        tfds.core.SplitGenerator(
            name=tfds.Split.TEST,
            num_shards=1,
            gen_kwargs=dict(split_name="tstid", **gen_kwargs)),
        tfds.core.SplitGenerator(
            name=tfds.Split.VALIDATION,
            num_shards=1,
            gen_kwargs=dict(split_name="valid", **gen_kwargs)),
    ]
282411
def _build_pcollection(self, pipeline, folder, split):
    """"""
    beam = tfds.core.lazy_imports.apache_beam

    split_type = self.builder_config.split_type
    filename = os.path.join(folder, "{}.tar.gz".format(split_type))


    def _extract_data(inputs):
      """Extracts files from the tar archives."""
      filename, split = inputs
      with tf.io.gfile.GFile(filename, "rb") as f:
        with tarfile.open(fileobj=f, mode="r") as tar:
          for tarinfo in tar:
            split_name = tarinfo.name.split("_")
            if len(split_name) > 2 and split_name[2] == split:
              buf = six.BytesIO()
              shutil.copyfileobj(tar.extractfile(tarinfo), buf)
              yield [tarinfo.name, buf.getvalue()]

    def _process_example(inputs):
      filename, data_string = inputs
      buf = six.BytesIO(data_string)
      buf.seek(0)
      data = np.load(buf)
      # Extract the images and convert to uint8. The reshape is required, see
      # https://github.com/deepmind/abstract-reasoning-matrices.
      all_images = np.uint8(data["image"].reshape(16, 160, 160, 1))
      return {
          "relation_structure_encoded": data["relation_structure_encoded"],
          "target": data["target"],
          "meta_target": data["meta_target"],
          "context": all_images[:8],
          "answers": all_images[8:],
          "filename": filename,
      }

    # Beam might fuse together the _extract_data and _process_example which
    # defeats the purpose of parallel processing. As a result, we reshard by
    # doing a GroupByKey on random keys, and then flattening again.
    def _add_random_keys(inputs):
      key = str(random.randrange(10**10))
      return key, inputs

    def _remove_keys(inputs):
      _, rows = inputs
      for row in rows:
        yield row

    return (pipeline
            | beam.Create([(filename, split)])
            | beam.FlatMap(_extract_data)
            | beam.Map(_add_random_keys)
            | beam.GroupByKey()
            | beam.FlatMap(_remove_keys)
            | beam.Map(_process_example))
751603
def _split_batches(self, data, batch_size):
        """"""
        for i in range(0, len(data), batch_size):
            yield data[i : i + batch_size]
779530
def writer(datasets, dataset_names, output_dir):
    """"""
    for dataset, dataset_name in zip(datasets, dataset_names):
        if dataset:
            filepath = output_dir + '/' + dataset_name + '.txt'
            with open(filepath, 'w+') as out_file:
                joined = '\n'.join(dataset)
                out_file.write(str(joined.encode('utf-8').decode('utf-8')))
                out_file.write('\n')
551645
def reducer_count(self, key, values):
        
        total = sum(values)
        metro, poi = key
        # group data by metro areas for final output    
        yield metro, (total, poi)
177208
def split_subsets(self, train_size:float, valid_size:float, seed=None) -> 'ItemLists':
        ""
        assert 0 < train_size < 1
        assert 0 < valid_size < 1
        assert train_size + valid_size <= 1.
        if seed is not None: np.random.seed(seed)
        n = len(self.items)
        rand_idx = np.random.permutation(range(n))
        train_cut, valid_cut = int(train_size * n), int(valid_size * n)
        return self.split_by_idxs(rand_idx[:train_cut], rand_idx[-valid_cut:])
1119924
def _split_generators(self, dl_manager):
    """"""
    # Download the full MNIST Database
    filenames = {
        "train_data": _MNIST_TRAIN_DATA_FILENAME,
        "train_labels": _MNIST_TRAIN_LABELS_FILENAME,
        "test_data": _MNIST_TEST_DATA_FILENAME,
        "test_labels": _MNIST_TEST_LABELS_FILENAME,
    }
    mnist_files = dl_manager.download_and_extract(
        {k: urllib.parse.urljoin(self.URL, v) for k, v in filenames.items()})

    # MNIST provides TRAIN and TEST splits, not a VALIDATION split, so we only
    # write the TRAIN and TEST splits to disk.
    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            num_shards=10,
            gen_kwargs=dict(
                num_examples=_TRAIN_EXAMPLES,
                data_path=mnist_files["train_data"],
                label_path=mnist_files["train_labels"],
            )),
        tfds.core.SplitGenerator(
            name=tfds.Split.TEST,
            num_shards=1,
            gen_kwargs=dict(
                num_examples=_TEST_EXAMPLES,
                data_path=mnist_files["test_data"],
                label_path=mnist_files["test_labels"],
            )),
    ]
103329
def summarize_features(features, num_shards=1):
  """"""
  if not common_layers.should_generate_summaries():
    return

  with tf.name_scope("input_stats"):
    for (k, v) in sorted(six.iteritems(features)):
      if (isinstance(v, tf.Tensor) and (v.get_shape().ndims > 1) and
          (v.dtype != tf.string)):
        tf.summary.scalar("%s_batch" % k, tf.shape(v)[0] // num_shards)
        tf.summary.scalar("%s_length" % k, tf.shape(v)[1])
        nonpadding = tf.to_float(tf.not_equal(v, 0))
        nonpadding_tokens = tf.reduce_sum(nonpadding)
        tf.summary.scalar("%s_nonpadding_tokens" % k, nonpadding_tokens)
        tf.summary.scalar("%s_nonpadding_fraction" % k,
                          tf.reduce_mean(nonpadding))
1126511
public void splitShards(int targetNumShards) {
    int origionalShardCount = getShardCount();
    int splitFactor = targetNumShards / origionalShardCount;
    if (splitFactor <= 1) {
      return;
    }
    List<List<String>> oldAllFiles = allFiles;
    shardCount = splitFactor * origionalShardCount;
    allFiles = initAllFiles(shardCount);
    int shard = 0;
    for (List<String> files : oldAllFiles) {
      if (!files.isEmpty()) {
        for (List<String> forShard : SplitUtil.split(files, splitFactor, false)) {
          addFilesToShard(shard++, forShard);
        }
      }
    }
  }