Create a transducer which produces non-overlapping batches.
967881
def batching(size):
    """"""

    if size < 1:
        raise ValueError("batching() size must be at least 1")

    def batching_transducer(reducer):
        return Batching(reducer, size)

    return batching_transducer
924975
def batch_iter(data, batch_size, num_epochs):
    """"""
    data = np.array(data)
    data_size = len(data)
    num_batches_per_epoch = int(len(data)/batch_size) + 1
    for epoch in range(num_epochs):
        # Shuffle the data at each epoch
        shuffle_indices = np.random.permutation(np.arange(data_size))
        shuffled_data = data[shuffle_indices]
        for batch_num in range(num_batches_per_epoch):
            start_index = batch_num * batch_size
            end_index = min((batch_num + 1) * batch_size, data_size)
            yield shuffled_data[start_index:end_index]
1113665
def next(self):
        """"""
        if not self.iter_next():
            raise StopIteration
        data = self.getdata()
        label = self.getlabel()
        # iter should stop when last batch is not complete
        if data[0].shape[0] != self.batch_size:
        # in this case, cache it for next epoch
            self._cache_data = data
            self._cache_label = label
            raise StopIteration
        return DataBatch(data=data, label=label, \
            pad=self.getpad(), index=None)
342241
def _create_rnn_architecture(self):
        """"""
        self.inputs = [tf.squeeze(i, [1]) for i in tf.split(
            axis=1, num_or_size_splits=self.num_steps, value=self.inputs)]
        outputs, state = tf.nn.rnn(
            self.cell, self.inputs, initial_state=self._init_state)

        output = tf.reshape(tf.concat(axis=1, values=outputs), [-1, self.num_hidden])
        softmax_w = tf.get_variable(
            "softmax_w", [self.num_hidden, self.vocab_size])
        softmax_b = tf.get_variable("softmax_b", [self.vocab_size])
        logits = tf.add(tf.matmul(output, softmax_w), softmax_b)
        loss = tf.nn.seq2seq.sequence_loss_by_example(
            [logits],
            [tf.reshape(self.input_labels, [-1])],
            [tf.ones([self.batch_size * self.num_steps])])

        self.cost = tf.div(tf.reduce_sum(loss), self.batch_size)
        self.final_state = state
342722
def shuffled_batches(self, batch_size):
        
        if batch_size >= self.size:
            yield self
        else:
            batch_splits = math_util.divide_ceiling(self.size, batch_size)
            indices = list(range(self.size))
            np.random.shuffle(indices)

            for sub_indices in np.array_split(indices, batch_splits):
                yield Transitions(
                    size=len(sub_indices),
                    environment_information=None,
                    # Dont use it in batches for a moment, can be uncommented later if needed
                    # environment_information=[info[sub_indices.tolist()] for info in self.environment_information]
                    transition_tensors={k: v[sub_indices] for k, v in self.transition_tensors.items()}
                    # extra_data does not go into batches
                )
1164824
def _prepare_batch(batch, device=None, non_blocking=False):
    
    x, y = batch
    return (convert_tensor(x, device=device, non_blocking=non_blocking),
            convert_tensor(y, device=device, non_blocking=non_blocking))
469535
def _batchify(self, batch_data, batch_label, start=0):
        """"""
        i = start
        batch_size = self.batch_size
        try:
            while i < batch_size:
                label, s = self.next_sample()
                data = self.imdecode(s)
                try:
                    self.check_valid_image([data])
                    label = self._parse_label(label)
                    data, label = self.augmentation_transform(data, label)
                    self._check_valid_label(label)
                except RuntimeError as e:
                    logging.debug('Invalid image, skipping:  %s', str(e))
                    continue
                for datum in [data]:
                    assert i < batch_size, 'Batch size must be multiples of augmenter output length'
                    batch_data[i] = self.postprocess_data(datum)
                    num_object = label.shape[0]
                    batch_label[i][0:num_object] = nd.array(label)
                    if num_object < batch_label[i].shape[0]:
                        batch_label[i][num_object:] = -1
                    i += 1
        except StopIteration:
            if not i:
                raise StopIteration

        return i
964822
def runMultiprocessNoiseExperiment(resultName, repeat, numWorkers,
                                   appendResults, **kwargs):
  
  experiments = [{}]
  for key, values in kwargs.items():
    if type(values) is list:
      newExperiments = []
      for experiment in experiments:
        for val in values:
          newExperiment = copy(experiment)
          newExperiment[key] = val
          newExperiments.append(newExperiment)
      experiments = newExperiments
    else:
      [a.__setitem__(key, values) for a in experiments]

  newExperiments = []
  for experiment in experiments:
    for _ in xrange(repeat):
      newExperiments.append(copy(experiment))
  experiments = newExperiments

  return runExperiments(experiments, resultName, numWorkers, appendResults)
716211
public BatchCSVRecord transform(BatchCSVRecord batch) {
        BatchCSVRecord batchCSVRecord = new BatchCSVRecord();
        List<List<Writable>> converted =  execute(toArrowWritables(toArrowColumnsString(
                bufferAllocator,transformProcess.getInitialSchema(),
                batch.getRecordsAsString()),
                transformProcess.getInitialSchema()),transformProcess);
        int numCols = converted.get(0).size();
        for (int row = 0; row < converted.size(); row++) {
            String[] values = new String[numCols];
            for (int i = 0; i < values.length; i++)
                values[i] = converted.get(row).get(i).toString();
            batchCSVRecord.add(new SingleCSVRecord(values));
        }

        return batchCSVRecord;

    }
1277457
def prepare_batch(self, batch: mx.io.DataBatch):
        
        self.module.prepare(batch)
5648
def _collate(self, batch):
        

        Y_batch = None
        if isinstance(batch[0], tuple):
            batch, Y_batch = list(zip(*batch))
            Y_batch = self._cuda(torch.Tensor(Y_batch))

        f_batch, v_batch = list(zip(*batch))

        f_batch, _ = pad_batch(f_batch, 0)
        v_batch, _ = pad_batch(v_batch, 0, type="float")

        f_batch = self._cuda(f_batch)
        v_batch = self._cuda(v_batch)

        if Y_batch is not None:
            return [f_batch, v_batch], Y_batch
        else:
            return [f_batch, v_batch]
248134
def _take_batch(self, min_records):
        ''''''

        if not self._buffer:
            return []

        enough_messages = len(self._buffer) >= min_records
        enough_time = time.time() - self.time_last_batch_sent >= self.batch_delay_seconds
        ready = enough_messages or enough_time

        if not ready:
            return []

        result = list(self._buffer)
        self._buffer.clear()
        return result
263105
def replace_negative_size_with_batch_size(shape, batch_size):
    """"""
    sl = []
    for d in shape.dim:
        if d < 0:
            # Negative size means batch size
            sl.append(batch_size)
        else:
            sl.append(d)
    out_shape = nnabla_pb2.Shape()
    out_shape.dim.extend(sl)
    return out_shape
927668
def default_batchify_fn(data):
    """"""
    if isinstance(data[0], nd.NDArray):
        return nd.stack(*data)
    elif isinstance(data[0], tuple):
        data = zip(*data)
        return [default_batchify_fn(i) for i in data]
    else:
        data = np.asarray(data)
        return nd.array(data, dtype=data.dtype)
954022
private void initBatcher(String appenderName) {
        MessageProcessor<LoggingEvent> messageProcessor = new MessageProcessor<LoggingEvent>() {
            @Override
            public void process(List<LoggingEvent> objects) {
                processLoggingEvents(objects);
            }
        };
        String batcherName = this.getClass().getName() + BATCHER_NAME_LIMITER
                + appenderName;
        batcher = BatcherFactory.createBatcher(batcherName, messageProcessor);
        batcher.setTarget(messageProcessor);

    }
552227
def batch(data, batch_size, batch_size_fn=None):
    """"""
    if batch_size_fn is None:
        def batch_size_fn(new, count, sofar):
            return count
    minibatch, size_so_far = [], 0
    for ex in data:
        minibatch.append(ex)
        size_so_far = batch_size_fn(ex, len(minibatch), size_so_far)
        if size_so_far == batch_size:
            yield minibatch
            minibatch, size_so_far = [], 0
        elif size_so_far > batch_size:
            yield minibatch[:-1]
            minibatch, size_so_far = minibatch[-1:], batch_size_fn(ex, 1, 0)
    if minibatch:
        yield minibatch
272836
def batch_gen(data, batch_size):
    '''
    
    '''
    data = data or []
    for i in range(0, len(data), batch_size):
        yield data[i:i + batch_size]