Returns the number of batches that ``dataset`` will be split into; if you want to track
        progress through the batch with the generator produced by ``__call__``, this could be
        useful.
196476
def get_num_batches(self, instances: Iterable[Instance]) -> int:
        
        if is_lazy(instances) and self._instances_per_epoch is None:
            # Unable to compute num batches, so just return 1.
            return 1
        elif self._instances_per_epoch is not None:
            return math.ceil(self._instances_per_epoch / self._batch_size)
        else:
            # Not lazy, so can compute the list length.
            return math.ceil(len(ensure_list(instances)) / self._batch_size)
306100
def next(self):
        """"""
        if self.curr_idx == len(self.idx):
            raise StopIteration
        i, j = self.idx[self.curr_idx]
        self.curr_idx += 1

        if self.major_axis == 1:
            data = self.nddata[i][j:j+self.batch_size].T
            label = self.ndlabel[i][j:j+self.batch_size].T
        else:
            data = self.nddata[i][j:j+self.batch_size]
            label = self.ndlabel[i][j:j+self.batch_size]

        return DataBatch([data], [label], pad=0,
                         bucket_key=self.buckets[i],
                         provide_data=[DataDesc(
                             name=self.data_name, shape=data.shape,
                             layout=self.layout)],
                         provide_label=[DataDesc(
                             name=self.label_name, shape=label.shape,
                             layout=self.layout)])
276366
def get_batch(sequence, size, start=0, endpoint=None, complete=False):
    

    batch = make_batch(sequence, size, start)

    return {
        "pagesize": batch.get_pagesize(),
        "next": batch.make_next_url(),
        "previous": batch.make_prev_url(),
        "page": batch.get_pagenumber(),
        "pages": batch.get_numpages(),
        "count": batch.get_sequence_length(),
        "items": make_items_for([b for b in batch.get_batch()],
                                endpoint, complete=complete),
    }
921402
def dataset_walker(datasets):
    
    for dataset in datasets:
        yield dataset, None
        for anc_ds in dataset.attrs.get('ancillary_variables', []):
            try:
                anc_ds.attrs
                yield anc_ds, dataset
            except AttributeError:
                continue
615069
def new_batch(self):
        
        i = next(self.slicer)
        return self.X_all[i], self.Y_all[i]
840826
def get_data_iters(dataset, batch_size, opt):
    """"""
    if dataset == 'mnist':
        train_data, val_data = get_mnist_iterator(batch_size, (1, 28, 28),
                                                  num_parts=kv.num_workers, part_index=kv.rank)
    elif dataset == 'cifar10':
        train_data, val_data = get_cifar10_iterator(batch_size, (3, 32, 32),
                                                    num_parts=kv.num_workers, part_index=kv.rank)
    elif dataset == 'imagenet':
        shape_dim = 299 if model_name == 'inceptionv3' else 224

        if not opt.data_dir:
            raise ValueError('Dir containing raw images in train/val is required for imagenet.'
                             'Please specify "--data-dir"')

        train_data, val_data = get_imagenet_iterator(opt.data_dir, batch_size,
                                                                opt.num_workers, shape_dim, opt.dtype)
    elif dataset == 'caltech101':
        train_data, val_data = get_caltech101_iterator(batch_size, opt.num_workers, opt.dtype)
    elif dataset == 'dummy':
        shape_dim = 299 if model_name == 'inceptionv3' else 224
        train_data, val_data = dummy_iterator(batch_size, (3, shape_dim, shape_dim))
    return train_data, val_data
1169798
def combine_calls_parallel(samples, run_parallel):
    
    batch_groups, extras = _group_by_batches(samples, _has_ensemble)
    out = []
    if batch_groups:
        processed = run_parallel("combine_calls", ((b, xs, xs[0]) for b, xs in batch_groups.items()))
        for batch_id, callinfo in processed:
            for data in batch_groups[batch_id]:
                data["variants"].insert(0, callinfo)
                out.append([data])
    return out + extras
887954
def _group_by_batches(samples, check_fn):
    
    batch_groups = collections.defaultdict(list)
    extras = []
    for data in [x[0] for x in samples]:
        if check_fn(data):
            batch_groups[multi.get_batch_for_key(data)].append(data)
        else:
            extras.append([data])
    return batch_groups, extras
861167
def batch_fun(dataset, training, shapes, target_names, num_devices,
              batch_size_per_device=32, batch_size=None, eval_batch_size=32,
              bucket_length=32, buckets=None,
              batch_shuffle_size=128, max_eval_length=None):
  """"""
  del target_names
  # Batch size is batch_size_per_device * num_devices unless given directly.
  batch_size = batch_size or batch_size_per_device * num_devices
  # If bucketing is not specified, check if target shapes are variable.
  cur_batch_size = batch_size if training else eval_batch_size
  # Make cur_batch_size divisible by num_devices.
  cur_batch_size = max(cur_batch_size // num_devices, 1) * num_devices
  # Create heuristic buckets is none are specified.
  if buckets is None:
    variable_target_shapes = False
    target_shape = shapes[1]
    for dim in target_shape:
      if dim is None:
        variable_target_shapes = True
    tf.logging.info("Heuristically setting bucketing to %s based on shapes "
                    "of target tensors." % variable_target_shapes)
    if variable_target_shapes:
      bucket_boundaries = [bucket_length // 4, bucket_length // 2,
                           bucket_length, bucket_length * 2,
                           bucket_length * 4, bucket_length * 8,
                           bucket_length * 16]
      # We will pad to boundaries which pads to bucket_boundary - 1: add 1 here.
      bucket_boundaries = [b + 1 for b in bucket_boundaries]
      if not training:
        max_eval_length = max_eval_length or bucket_length * 32
        bucket_boundaries[-1] = max_eval_length
      bucket_batch_sizes = [cur_batch_size * 4, cur_batch_size * 2,
                            cur_batch_size, cur_batch_size // 2,
                            cur_batch_size // 4, cur_batch_size // 8,
                            cur_batch_size // 16, 1]
      if not training:
        bucket_batch_sizes[-2] = cur_batch_size // max_eval_length
      # Make batch sizes divisible by num_devices.
      bucket_batch_sizes = [max(b // num_devices, 1) * num_devices
                            for b in bucket_batch_sizes]
      buckets = (bucket_boundaries, bucket_batch_sizes)

  if buckets:
    tf.logging.info("Bucketing with buckets %s." % str(buckets))
    def example_length(_, target):
      return tf.shape(target)[0]
    boundaries, batch_sizes = buckets
    dataset = dataset.apply(tf.data.experimental.bucket_by_sequence_length(
        example_length, boundaries, batch_sizes,
        pad_to_bucket_boundary=True))
  else:
    dataset = dataset.padded_batch(cur_batch_size, shapes)
  if training:
    return dataset.shuffle(batch_shuffle_size)
  return dataset
157657
def n_chunks(self):
        
        return self._data_source.n_chunks(self.chunksize, stride=self.stride, skip=self.skip)
552227
def batch(data, batch_size, batch_size_fn=None):
    """"""
    if batch_size_fn is None:
        def batch_size_fn(new, count, sofar):
            return count
    minibatch, size_so_far = [], 0
    for ex in data:
        minibatch.append(ex)
        size_so_far = batch_size_fn(ex, len(minibatch), size_so_far)
        if size_so_far == batch_size:
            yield minibatch
            minibatch, size_so_far = [], 0
        elif size_so_far > batch_size:
            yield minibatch[:-1]
            minibatch, size_so_far = minibatch[-1:], batch_size_fn(ex, 1, 0)
    if minibatch:
        yield minibatch
675128
def get_next_batch(self):
        
        number_of_items = self.read_option('number_of_items')
        for i in range(0, self.batch_size):
            to_read = self.last_read + 1
            if to_read >= number_of_items:
                self.finished = True
                break
            else:
                item = BaseRecord()
                self.last_read = to_read
                item['key'] = self.last_read
                item['country_code'] = random.choice(self.country_codes)
                item['state'] = random.choice(self.states)
                item['city'] = random.choice(self.cities)
                item['value'] = random.randint(0, 10000)
                self.increase_read()
                self.last_position['last_read'] = self.last_read
                yield item
        self.logger.debug('Done reading batch')
1020595
def generate_batches(sequence, batch_len=1, allow_partial=True, ignore_errors=True, verbosity=1):
    
    it = iter(sequence)
    last_value = False
    # An exception will be thrown by `.next()` here and caught in the loop that called this iterator/generator
    while not last_value:
        batch = []
        for n in range(batch_len):
            try:
                batch += (next(it),)
            except StopIteration:
                last_value = True
                if batch:
                    break
                else:
                    raise StopIteration
            except Exception:
                # 'Error: new-line character seen in unquoted field -
                # do you need to open the file in universal-newline mode?'
                if verbosity > 0:
                    print_exc()
                if not ignore_errors:
                    raise
        yield batch
965436
def get_training_data(batch_size):
    
    return gluon.data.DataLoader(
        CIFAR10(train=True, transform=transformer),
        batch_size=batch_size, shuffle=True, last_batch='discard')
1187956
def previous(self):
        
        if self.start - self.size < 0:
            result = None
        else:
            result = Batch(self.start - self.size, self.size, self.total_size)
        return result
848804
def next(self) -> mx.io.DataBatch:
        
        if not self.iter_next():
            raise StopIteration

        i, j = self.batch_indices[self.curr_batch_index]
        self.curr_batch_index += 1

        batch_size = self.bucket_batch_sizes[i].batch_size
        source = self.data.source[i][j:j + batch_size]
        target = self.data.target[i][j:j + batch_size]
        if self.preload_features:
            loaded_source = []  # type: List[np.ndarray]
            for k in source:
                loaded_source.append(self.loaded_source[k])
        else:
            loaded_source = self.data_loader(source)
        # zero pad features if not agree with expected shape
        loaded_source = zero_pad_features(loaded_source, self.image_size)
        loaded_source = mx.nd.array(loaded_source)

        label = [self.data.label[i][j:j + batch_size]]

        provide_data = [mx.io.DataDesc(name=self.source_data_name, shape=loaded_source.shape, layout=C.BATCH_MAJOR_IMAGE)]
        if self.with_text:
            provide_data += [mx.io.DataDesc(name=self.target_data_name, shape=target.shape, layout=C.BATCH_MAJOR)]
        provide_label = [mx.io.DataDesc(name=n, shape=x.shape, layout=C.BATCH_MAJOR) for n, x in
                         zip(self.label_names, label)]

        data = [loaded_source]
        if self.with_text:
            data += [target]
        return mx.io.DataBatch(data, label,
                               pad=0, index=None, bucket_key=self.buckets[i],
                               provide_data=provide_data, provide_label=provide_label)
1373770
def gen_batches(data, batch_size):
    
    data = np.array(data)

    for i in range(0, data.shape[0], batch_size):
        yield data[i:i + batch_size]