Return train and eval batches with input name and shape.
188583
def _train_and_eval_batches(dataset, data_dir, input_name, num_devices):
  """"""
  (train_data, eval_data, features_info, keys) = train_and_eval_dataset(
      dataset, data_dir)
  input_names, target_names = keys[0], keys[1]
  train_batches = shuffle_and_batch_data(
      train_data, target_names, features_info, training=True,
      num_devices=num_devices)
  train_eval_batches = shuffle_and_batch_data(  # Data for eval-on-train.
      train_data, target_names, features_info, training=False,
      num_devices=num_devices)
  eval_batches = shuffle_and_batch_data(
      eval_data, target_names, features_info, training=False,
      num_devices=num_devices)
  input_name = input_name or input_names[0]
  input_shape = features_info[input_name].shape
  return (train_batches, train_eval_batches, eval_batches,
          input_name, list(input_shape))
157875
def calculate_reshape_output_shapes(operator):
    '''
    
    '''
    check_input_and_output_numbers(operator, input_count_range=1, output_count_range=1)
    check_input_and_output_types(operator, good_input_types=[FloatTensorType])

    params = operator.raw_operator.reshape

    output_shape = list(int(i) for i in params.targetShape)

    if len(output_shape) == 3:
        output_shape = [operator.inputs[0].type.shape[0]] + output_shape

    operator.outputs[0].type.shape = output_shape
203754
def calculate_feature_vectorizer_output_shapes(operator):
    '''
    
    '''
    check_input_and_output_numbers(operator, input_count_range=[1, None], output_count_range=1)
    check_input_and_output_types(operator, good_input_types=[FloatTensorType, Int64TensorType, FloatType, Int64Type])

    if any(len(variable.type.shape) != 2 for variable in operator.inputs):
        raise RuntimeError('Input(s) must be 2-D tensor(s)')

    # Find the first batch size which is not unknown
    N = 'None'
    for variable in operator.inputs:
        if variable.type.shape[0] != 'None':
            N = variable.type.shape[0]
            break
    for variable in operator.inputs:
        if variable.type.shape[0] not in ['None', N]:
            raise RuntimeError('The batch dimensions should be the same to all input tensors.')

    C = sum(info.inputDimensions for info in operator.raw_operator.featureVectorizer.inputList)

    # Currently, we only expect numerical inputs. If both of integers and floats exist, we may convert integers into
    # floats before concatenating them. Thus, the output type is integer-like only if all inputs are integer-like.
    doc_string = operator.outputs[0].type.doc_string
    if all(isinstance(variable.type, (Int64TensorType, Int64Type)) for variable in operator.inputs):
        operator.outputs[0].type = Int64TensorType([N, C], doc_string=doc_string)
    elif isinstance(operator.inputs[0].type, (FloatTensorType, FloatType)):
        operator.outputs[0].type = FloatTensorType([N, C], doc_string=doc_string)
    else:
        raise ValueError('Unsupported input type: %s' % type(operator.inputs[0].type))
307121
def transpose(self, name=None):
    """"""
    if name is None:
      name = self.module_name + "_transpose"
    return BatchReshape(shape=lambda: self.input_shape,
                        preserve_dims=self._preserve_dims,
                        name=name)
213044
def get_cur_batch(items):
    
    batches = []
    for data in items:
        batch = tz.get_in(["metadata", "batch"], data, [])
        batches.append(set(batch) if isinstance(batch, (list, tuple)) else set([batch]))
    combo_batches = reduce(lambda b1, b2: b1.intersection(b2), batches)
    if len(combo_batches) == 1:
        return combo_batches.pop()
    elif len(combo_batches) == 0:
        return None
    else:
        raise ValueError("Found multiple overlapping batches: %s -- %s" % (combo_batches, batches))
119926
def _chunked_selector_output_shape(  # pylint: disable=invalid-name
    input_shapes, selector=None, **unused_kwargs):
  """"""
  # Read the main function below first, the shape logic just follows the ops.
  selector = selector or (lambda x: [] if x < 1 else [x-1])
  triples, _ = zip(*input_shapes)
  (query_shapes, key_shapes, value_shapes) = zip(*triples)
  result = []
  for i in range(len(input_shapes)):
    selected = selector(i)
    cur_key_shape, cur_value_shape = key_shapes[i], value_shapes[i]
    # Since keys and values are [batch, length, depth] we concatenate on axis=1.
    new_key_len = sum([key_shapes[j][1] for j in selected]) + cur_key_shape[1]
    new_key_shape = (cur_key_shape[0], new_key_len, cur_key_shape[2])
    new_value_len = sum(
        [value_shapes[j][1] for j in selected]) + cur_value_shape[1]
    new_value_shape = (cur_value_shape[0], new_value_len, cur_value_shape[2])
    # Masks are (1, query-len, key-len).
    new_mask_shape = (1, query_shapes[i][1], new_key_len)
    new_shape = ((query_shapes[i], new_key_shape, new_value_shape),
                 new_mask_shape)
    result.append(new_shape)
  return tuple(result)
1060418
def one_batch(self, ds_type:DatasetType=DatasetType.Train, detach:bool=True, denorm:bool=True, cpu:bool=True)->Collection[Tensor]:
        ""
        dl = self.dl(ds_type)
        w = self.num_workers
        self.num_workers = 0
        try:     x,y = next(iter(dl))
        finally: self.num_workers = w
        if detach: x,y = to_detach(x,cpu=cpu),to_detach(y,cpu=cpu)
        norm = getattr(self,'norm',False)
        if denorm and norm:
            x = self.denorm(x)
            if norm.keywords.get('do_y',False): y = self.denorm(y, do_x=True)
        return x,y
939716
def main(_):
  """"""
  batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]
  for filenames, images in load_images(FLAGS.input_dir, batch_shape):
    save_images(images, filenames, FLAGS.output_dir)
1165471
def main(_):
  """"""
  eps = FLAGS.max_epsilon / 255.0
  batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]

  with tf.Graph().as_default():
    x_input = tf.placeholder(tf.float32, shape=batch_shape)
    noisy_images = x_input + eps * tf.sign(tf.random_normal(batch_shape))
    x_output = tf.clip_by_value(noisy_images, 0.0, 1.0)

    with tf.Session(FLAGS.master) as sess:
      for filenames, images in load_images(FLAGS.input_dir, batch_shape):
        out_images = sess.run(x_output, feed_dict={x_input: images})
        save_images(out_images, filenames, FLAGS.output_dir)
582938
def cbow_batch(centers, contexts, num_tokens, dtype, index_dtype):
    """"""
    contexts_data, contexts_row, contexts_col = contexts
    centers = mx.nd.array(centers, dtype=index_dtype)
    contexts = mx.nd.sparse.csr_matrix(
        (contexts_data, (contexts_row, contexts_col)),
        dtype=dtype, shape=(len(centers), num_tokens))  # yapf: disable
    return centers, contexts
850289
def batch(batch_size, items):
    ""
    items = list(items)
    if batch_size is None:
        return [items]
    MISSING = object()
    padded_items = items + [MISSING] * (batch_size - 1)
    groups = zip(*[padded_items[i::batch_size] for i in range(batch_size)])
    return [[item for item in group if item != MISSING] for group in groups]
488426
def gather_blocks_2d(x, indices):
  """"""
  x_shape = common_layers.shape_list(x)
  x = reshape_range(x, 2, 4, [tf.reduce_prod(x_shape[2:4])])
  # [length, batch, heads, dim]
  x_t = tf.transpose(x, [2, 0, 1, 3])
  x_new = tf.gather(x_t, indices)
  # returns [batch, heads, num_blocks, block_length ** 2, dim]
  return tf.transpose(x_new, [2, 3, 0, 1, 4])
409138
def train_input_fn(params):
  """"""
  file_pattern = os.path.join(getattr(params, "data_dir", ""), "*encoded-train*")
  return _read_and_batch_from_files(
      file_pattern, params.batch_size, params.max_length, params.num_cpu_cores,
      shuffle=True, repeat=params.repeat_dataset)
992162
def samples_to_batches(samples: Iterable, batch_size: int):
        """"""
        it = iter(samples)
        while True:
            with suppress(StopIteration):
                batch_in, batch_out = [], []
                for i in range(batch_size):
                    sample_in, sample_out = next(it)
                    batch_in.append(sample_in)
                    batch_out.append(sample_out)
            if not batch_in:
                raise StopIteration
            yield np.array(batch_in), np.array(batch_out)
1017175
def train_supervised(problem, model_name, hparams, data_dir, output_dir,
                     train_steps, eval_steps, local_eval_frequency=None,
                     schedule="continuous_train_and_eval"):
  """"""
  if local_eval_frequency is None:
    local_eval_frequency = FLAGS.local_eval_frequency

  exp_fn = trainer_lib.create_experiment_fn(
      model_name, problem, data_dir, train_steps, eval_steps,
      min_eval_frequency=local_eval_frequency
  )
  run_config = trainer_lib.create_run_config(model_name, model_dir=output_dir)
  exp = exp_fn(run_config, hparams)
  getattr(exp, schedule)()
746584
def prepare_data(problem, hparams, params, config):
  """"""
  input_fn = problem.make_estimator_input_fn(
      tf.estimator.ModeKeys.EVAL, hparams, force_repeat=True)
  dataset = input_fn(params, config)
  features, _ = dataset.make_one_shot_iterator().get_next()
  inputs, labels = features["targets"], features["inputs"]
  inputs = tf.to_float(inputs)
  input_shape = inputs.shape.as_list()
  inputs = tf.reshape(inputs, [hparams.batch_size] + input_shape[1:])
  labels = tf.reshape(labels, [hparams.batch_size])
  return inputs, labels, features
861167
def batch_fun(dataset, training, shapes, target_names, num_devices,
              batch_size_per_device=32, batch_size=None, eval_batch_size=32,
              bucket_length=32, buckets=None,
              batch_shuffle_size=128, max_eval_length=None):
  """"""
  del target_names
  # Batch size is batch_size_per_device * num_devices unless given directly.
  batch_size = batch_size or batch_size_per_device * num_devices
  # If bucketing is not specified, check if target shapes are variable.
  cur_batch_size = batch_size if training else eval_batch_size
  # Make cur_batch_size divisible by num_devices.
  cur_batch_size = max(cur_batch_size // num_devices, 1) * num_devices
  # Create heuristic buckets is none are specified.
  if buckets is None:
    variable_target_shapes = False
    target_shape = shapes[1]
    for dim in target_shape:
      if dim is None:
        variable_target_shapes = True
    tf.logging.info("Heuristically setting bucketing to %s based on shapes "
                    "of target tensors." % variable_target_shapes)
    if variable_target_shapes:
      bucket_boundaries = [bucket_length // 4, bucket_length // 2,
                           bucket_length, bucket_length * 2,
                           bucket_length * 4, bucket_length * 8,
                           bucket_length * 16]
      # We will pad to boundaries which pads to bucket_boundary - 1: add 1 here.
      bucket_boundaries = [b + 1 for b in bucket_boundaries]
      if not training:
        max_eval_length = max_eval_length or bucket_length * 32
        bucket_boundaries[-1] = max_eval_length
      bucket_batch_sizes = [cur_batch_size * 4, cur_batch_size * 2,
                            cur_batch_size, cur_batch_size // 2,
                            cur_batch_size // 4, cur_batch_size // 8,
                            cur_batch_size // 16, 1]
      if not training:
        bucket_batch_sizes[-2] = cur_batch_size // max_eval_length
      # Make batch sizes divisible by num_devices.
      bucket_batch_sizes = [max(b // num_devices, 1) * num_devices
                            for b in bucket_batch_sizes]
      buckets = (bucket_boundaries, bucket_batch_sizes)

  if buckets:
    tf.logging.info("Bucketing with buckets %s." % str(buckets))
    def example_length(_, target):
      return tf.shape(target)[0]
    boundaries, batch_sizes = buckets
    dataset = dataset.apply(tf.data.experimental.bucket_by_sequence_length(
        example_length, boundaries, batch_sizes,
        pad_to_bucket_boundary=True))
  else:
    dataset = dataset.padded_batch(cur_batch_size, shapes)
  if training:
    return dataset.shuffle(batch_shuffle_size)
  return dataset