Batch - the most atomic unit of learning.

        For this reinforforcer, that involves:

        1. Roll out environment and store out experience in the buffer
        2. Sample the buffer and train the algo on sample batch
488086
def train_batch(self, batch_info: BatchInfo) -> None:
        
        # For each reinforcer batch:

        # 1. Roll out environment and store out experience in the buffer
        self.roll_out_and_store(batch_info)

        # 2. Sample the buffer and train the algo on sample batch
        self.train_on_replay_memory(batch_info)
1228671
def initialize(self, params, repetition):
    
    self.name = params["name"]
    self.dataDir = params.get("datadir", "data")
    self.seed = params.get("seed", 42) + repetition

    torch.manual_seed(self.seed)
    np.random.seed(self.seed)
    random.seed(self.seed)

    # Training
    self.epochs = params.get("epochs", 1)
    self.batch_size = params.get("batch_size", 128)
    self.batches_in_epoch = params.get("batches_in_epoch", 100000)
    self.first_epoch_batch_size = params.get("first_epoch_batch_size",
                                             self.batch_size)
    self.batches_in_first_epoch = params.get("batches_in_first_epoch",
                                             self.batches_in_epoch)

    # Testing
    self.test_batch_size = params.get("test_batch_size", 1000)
    self.test_batches_in_epoch = params.get("test_batches_in_epoch", 100000)
    self.noise_values = map(float,
                            params.get("noise_values", "0.0, 0.1").split(", "))
    self.best_noise_score = 0.0
    self.best_epoch = -1

    # Optimizer
    self.optimizer_class = eval(params.get("optimizer", "torch.optim.SGD"))
    self.lr = params.get("learning_rate", 0.05)
    self.momentum = params.get("momentum", 0.9)
    self.weight_decay = params.get("weight_decay", 0.0)
    self.optimizer_params = eval(params.get("optimizer_params", "{}"))
    self.lr_scheduler_gamma = params.get("lr_scheduler_gamma", 0.9)
    self.loss_function = eval(params.get("loss_function",
                                         "torch.nn.functional.cross_entropy"))

    # Network parameters
    self.conv1_sparsity = params.get("conv1_sparsity", 1.0)
    self.network_type = params.get("network_type", "sparse")
    self.growth_rate = params.get("growth_rate", 12)
    self.nblocks = map(int,
                       params.get("nblocks", "6, 12, 24, 16").split(", "))
    self.k_inference_factor = params.get("k_inference_factor", 1.5)

    self.dense_sparsities = map(float,
                                params.get("dense_sparsities",
                                           "1.0, 1.0, 1.0, 1.0").split(", "))
    self.transition_sparsities = map(float,
                                     params.get("transition_sparsities",
                                                "0.1, 0.1, 0.2").split(", "))
    self.linear_sparsity = params.get("linear_sparsity", 0.0)
    self.linear_weight_sparsity = params.get("linear_weight_sparsity", 0.3)
    self.linear_n = params.get("linear_n", 500)
    self.avg_pool_size = params.get("avg_pool_size", 2)
    self.dense_c1_out_planes = params.get("dense_c1_out_planes", 4*self.growth_rate)
1312918
def write_batch(self, batch):
        
        for item in batch:
            self.write_buffer.buffer(item)
            key = self.write_buffer.get_key_from_item(item)
            if self.write_buffer.should_write_buffer(key):
                self._write_current_buffer_for_group_key(key)
            self.increment_written_items()
            self._check_items_limit()
917444
def _create_batches(self, X, batch_size, shuffle_data=True):
        
        if shuffle_data:
            X = shuffle(X)

        if batch_size > X.shape[0]:
            batch_size = X.shape[0]

        max_x = int(np.ceil(X.shape[0] / batch_size))
        X = np.resize(X, (max_x, batch_size, X.shape[-1]))

        return X
1113665
def next(self):
        """"""
        if not self.iter_next():
            raise StopIteration
        data = self.getdata()
        label = self.getlabel()
        # iter should stop when last batch is not complete
        if data[0].shape[0] != self.batch_size:
        # in this case, cache it for next epoch
            self._cache_data = data
            self._cache_label = label
            raise StopIteration
        return DataBatch(data=data, label=label, \
            pad=self.getpad(), index=None)
188583
def _train_and_eval_batches(dataset, data_dir, input_name, num_devices):
  """"""
  (train_data, eval_data, features_info, keys) = train_and_eval_dataset(
      dataset, data_dir)
  input_names, target_names = keys[0], keys[1]
  train_batches = shuffle_and_batch_data(
      train_data, target_names, features_info, training=True,
      num_devices=num_devices)
  train_eval_batches = shuffle_and_batch_data(  # Data for eval-on-train.
      train_data, target_names, features_info, training=False,
      num_devices=num_devices)
  eval_batches = shuffle_and_batch_data(
      eval_data, target_names, features_info, training=False,
      num_devices=num_devices)
  input_name = input_name or input_names[0]
  input_shape = features_info[input_name].shape
  return (train_batches, train_eval_batches, eval_batches,
          input_name, list(input_shape))
491374
def train_batch(self, batch_info, data, target):
        
        batch_info.optimizer.zero_grad()
        loss = self.feed_batch(batch_info, data, target)
        loss.backward()

        if self.max_grad_norm is not None:
            batch_info['grad_norm'] = torch.nn.utils.clip_grad_norm_(
                filter(lambda p: p.requires_grad, self.model.parameters()),
                max_norm=self.max_grad_norm
            )

        batch_info.optimizer.step()
349286
def train(*tf_records: "Records to train on"):
    """"""
    tf.logging.set_verbosity(tf.logging.INFO)
    estimator = dual_net.get_estimator()

    effective_batch_size = FLAGS.train_batch_size
    if FLAGS.use_tpu:
        effective_batch_size *= FLAGS.num_tpu_cores

    if FLAGS.use_tpu:
        if FLAGS.use_bt:
            def _input_fn(params):
                games = bigtable_input.GameQueue(
                    FLAGS.cbt_project, FLAGS.cbt_instance, FLAGS.cbt_table)
                games_nr = bigtable_input.GameQueue(
                    FLAGS.cbt_project, FLAGS.cbt_instance, FLAGS.cbt_table + '-nr')
                return preprocessing.get_tpu_bt_input_tensors(
                    games,
                    games_nr,
                    params['batch_size'],
                    number_of_games=FLAGS.window_size,
                    random_rotation=True)
        else:
            def _input_fn(params):
                return preprocessing.get_tpu_input_tensors(
                    params['batch_size'],
                    tf_records,
                    random_rotation=True)
        # Hooks are broken with TPUestimator at the moment.
        hooks = []
    else:
        def _input_fn():
            return preprocessing.get_input_tensors(
                FLAGS.train_batch_size,
                tf_records,
                filter_amount=FLAGS.filter_amount,
                shuffle_buffer_size=FLAGS.shuffle_buffer_size,
                random_rotation=True)

        hooks = [UpdateRatioSessionHook(FLAGS.work_dir),
                 EchoStepCounterHook(output_dir=FLAGS.work_dir)]

    steps = FLAGS.steps_to_train
    logging.info("Training, steps = %s, batch = %s -> %s examples",
                 steps or '?', effective_batch_size,
                 (steps * effective_batch_size) if steps else '?')

    if FLAGS.use_bt:
        games = bigtable_input.GameQueue(
            FLAGS.cbt_project, FLAGS.cbt_instance, FLAGS.cbt_table)
        if not games.read_wait_cell():
            games.require_fresh_games(20000)
        latest_game = games.latest_game_number
        index_from = max(latest_game, games.read_wait_cell())
        print("== Last game before training:", latest_game, flush=True)
        print("== Wait cell:", games.read_wait_cell(), flush=True)

    try:
        estimator.train(_input_fn, steps=steps, hooks=hooks)
        if FLAGS.use_bt:
            bigtable_input.set_fresh_watermark(games, index_from,
                                               FLAGS.window_size)
    except:
        if FLAGS.use_bt:
            games.require_fresh_games(0)
        raise
399669
def _update_batches_if_needed(self)->None:
        ""
        if self.learn.data.valid_dl is None: return # Running learning rate finder, so return
        update_batches = self.data is not self.learn.data
        if not update_batches: return
        self.data = self.learn.data
        self.trn_batch = self._get_new_batch(ds_type=DatasetType.Train)
        self.val_batch = self._get_new_batch(ds_type=DatasetType.Valid)
1164824
def _prepare_batch(batch, device=None, non_blocking=False):
    
    x, y = batch
    return (convert_tensor(x, device=device, non_blocking=non_blocking),
            convert_tensor(y, device=device, non_blocking=non_blocking))
1127197
def each(&block)
      if @iterate == :each_row
        @batched ? self.each_row_batch(&block) : self.each_row(&block)
      elsif @iterate == :each_array
        @batched ? self.each_array_batch(&block) : self.each_array(&block)
      else
        @batched ? self.each_instance_batch(@type, &block) : self.each_instance(@type, &block)
      end
    end
1123659
def _batch(self):
        '''
        
        '''
        self.batch = '%s/%s.treeliker' % (self.tmpdir, self.basename)

        commands = []
        if not self.test_dataset:
            commands.append('set(output_type, single)')
            commands.append("set(examples, '%s.txt')" % self.basename)
        else:
            commands.append('set(output_type, train_test)')
            commands.append("set(train_set, '%s.txt')" % self.basename)
            commands.append("set(test_set, '%s_test.txt')" % self.basename)
        commands.append('set(template, %s)' % self.template)

        if not self.test_dataset:
            commands.append('set(output, %s.arff)' % self.basename)
        else:
            commands.append('set(output, conversion)')

        # Optional settings
        for key, val in self.settings.items():
            if val not in [None, '']:
                commands.append('set(%s, %s)' % (key, str(val)))

        commands.append('work(yes)')

        with open(self.batch, 'w') as f:
            f.write('\n'.join(commands))
1214054
def calculate(self, batch_info):
        
        value = self._value_function(batch_info)
        self.buffer += value
626724
def train_on_replay_memory(self, batch_info):
        
        self.model.train()

        # Algo will aggregate data into this list:
        batch_info['sub_batch_data'] = []

        for i in range(self.settings.training_rounds):
            sampled_rollout = self.env_roller.sample(batch_info, self.model, self.settings.training_steps)

            batch_result = self.algo.optimizer_step(
                batch_info=batch_info,
                device=self.device,
                model=self.model,
                rollout=sampled_rollout.to_device(self.device)
            )

            self.env_roller.update(rollout=sampled_rollout, batch_info=batch_result)

            batch_info['sub_batch_data'].append(batch_result)

        batch_info.aggregate_key('sub_batch_data')
1246104
def batch(input_iter, batch_size=32):
  """"""
  input_iter = iter(input_iter)
  next_ = list(itertools.islice(input_iter, batch_size))
  while next_:
    yield next_
    next_ = list(itertools.islice(input_iter, batch_size))
584612
def sample(self, batch_size, batch_idxs=None):
        
        # It is not possible to tell whether the first state in the memory is terminal, because it
        # would require access to the "terminal" flag associated to the previous state. As a result
        # we will never return this first state (only using `self.terminals[0]` to know whether the
        # second state is terminal).
        # In addition we need enough entries to fill the desired window length.
        assert self.nb_entries >= self.window_length + 2, 'not enough entries in the memory'

        if batch_idxs is None:
            # Draw random indexes such that we have enough entries before each index to fill the
            # desired window length.
            batch_idxs = sample_batch_indexes(
                self.window_length, self.nb_entries - 1, size=batch_size)
        batch_idxs = np.array(batch_idxs) + 1
        assert np.min(batch_idxs) >= self.window_length + 1
        assert np.max(batch_idxs) < self.nb_entries
        assert len(batch_idxs) == batch_size

        # Create experiences
        experiences = []
        for idx in batch_idxs:
            terminal0 = self.terminals[idx - 2]
            while terminal0:
                # Skip this transition because the environment was reset here. Select a new, random
                # transition and use this instead. This may cause the batch to contain the same
                # transition twice.
                idx = sample_batch_indexes(self.window_length + 1, self.nb_entries, size=1)[0]
                terminal0 = self.terminals[idx - 2]
            assert self.window_length + 1 <= idx < self.nb_entries

            # This code is slightly complicated by the fact that subsequent observations might be
            # from different episodes. We ensure that an experience never spans multiple episodes.
            # This is probably not that important in practice but it seems cleaner.
            state0 = [self.observations[idx - 1]]
            for offset in range(0, self.window_length - 1):
                current_idx = idx - 2 - offset
                assert current_idx >= 1
                current_terminal = self.terminals[current_idx - 1]
                if current_terminal and not self.ignore_episode_boundaries:
                    # The previously handled observation was terminal, don't add the current one.
                    # Otherwise we would leak into a different episode.
                    break
                state0.insert(0, self.observations[current_idx])
            while len(state0) < self.window_length:
                state0.insert(0, zeroed_observation(state0[0]))
            action = self.actions[idx - 1]
            reward = self.rewards[idx - 1]
            terminal1 = self.terminals[idx - 1]

            # Okay, now we need to create the follow-up state. This is state0 shifted on timestep
            # to the right. Again, we need to be careful to not include an observation from the next
            # episode if the last state is terminal.
            state1 = [np.copy(x) for x in state0[1:]]
            state1.append(self.observations[idx])

            assert len(state0) == self.window_length
            assert len(state1) == len(state0)
            experiences.append(Experience(state0=state0, action=action, reward=reward,
                                          state1=state1, terminal1=terminal1))
        assert len(experiences) == batch_size
        return experiences
773946
def learn(self, numEpochs, batchsize):
        """"""
        for epoch in range(numEpochs):
            print('epoch %d' % epoch)
            indexes = np.random.permutation(self.trainsize)
            for i in range(0, self.trainsize, batchsize):
                x = Variable(self.x_train[indexes[i: i + batchsize]])
                t = Variable(self.y_train[indexes[i: i + batchsize]])
                self.optimizer.update(self.model, x, t)