Create a batch for SG training objective.
556752
def skipgram_batch(centers, contexts, num_tokens, dtype, index_dtype):
    """"""
    contexts = mx.nd.array(contexts[2], dtype=index_dtype)
    indptr = mx.nd.arange(len(centers) + 1)
    centers = mx.nd.array(centers, dtype=index_dtype)
    centers_csr = mx.nd.sparse.csr_matrix(
        (mx.nd.ones(centers.shape), centers, indptr), dtype=dtype,
        shape=(len(centers), num_tokens))
    return centers_csr, contexts, centers
933549
def batch_step(self, batch_idx=None):
        
        if batch_idx is None:
            batch_idx = self.last_batch_idx + 1
        self.last_batch_idx = batch_idx
        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
            param_group['lr'] = lr
1048079
def make_batch(size, graph):
        
        i = iter(graph)
        chunk = list(islice(i, size))
        while chunk:
            yield chunk
            chunk = list(islice(i, size))
593029
def build_batch(cls, size, **kwargs):
        
        return [cls.build(**kwargs) for _ in range(size)]
595442
def initialize(self, params, repetition):
    
    self.name = params["name"]
    self.dataDir = params.get("datadir", "data")
    self.seed = params.get("seed", 42) + repetition

    torch.manual_seed(self.seed)
    np.random.seed(self.seed)
    random.seed(self.seed)

    # Training
    self.epochs = params.get("epochs", 1)
    self.batch_size = params.get("batch_size", 16)
    self.batches_in_epoch = params.get("batches_in_epoch", 60000)
    self.first_epoch_batch_size = params.get("first_epoch_batch_size", self.batch_size)
    self.batches_in_first_epoch = params.get("batches_in_first_epoch", self.batches_in_epoch)

    # Testing
    self.test_batch_size = params.get("test_batch_size", 1000)

    # Optimizer
    self.optimizer_class = eval(params.get("optimizer", "torch.optim.SGD"))
    self.optimizer_params = eval(params.get("optimizer_params", "{}"))
    self.lr_scheduler_class = eval(params.get("lr_scheduler", None))
    self.lr_scheduler_params = eval(params.get("lr_scheduler_params", "{}"))
    self.loss_function = eval(params.get("loss_function", "torch.nn.functional.nll_loss"))

    # CNN parameters
    c, h, w = map(int, params.get("input_shape", "1_28_28").split("_"))
    self.in_channels = c
    self.out_channels = map(int, params.get("out_channels", "30_30").split("_"))
    self.kernel_size = map(int, params.get("kernel_size", "5_5").split("_"))
    self.stride = map(int, params.get("stride", "1_1").split("_"))
    self.padding = map(int, params.get("padding", "0_0").split("_"))

    # Compute Flatten CNN output len
    self.maxpool = []
    self.maxpool.append(
      ((w + 2 * self.padding[0] - self.kernel_size[0]) // self.stride[0] + 1) // 2)
    self.maxpool.append(
      ((self.maxpool[0] + 2 * self.padding[1] - self.kernel_size[1]) // self.stride[1] + 1) // 2)

    self.cnn_output_len = [self.maxpool[i] * self.maxpool[i] * self.out_channels[i]
                           for i in range(len(self.maxpool))]

    # Linear parameteers
    self.n = params.get("n", 1000)
    self.output_size = params.get("output_size", 10)

    # Sparse parameters
    if "c1_k" in params:
      self.cnn_k = map(int, params["c1_k"].split("_"))
    else:
      self.cnn_k = self.cnn_output_len

    self.k = params.get("k", self.n)
    self.k_inference_factor = params.get("k_inference_factor", 1.0)
    self.boost_strength = params.get("boost_strength", 1.0)
    self.boost_strength_factor = params.get("boost_strength_factor", 1.0)
    self.weight_sparsity = params.get("weight_sparsity", 1.0)
    self.weight_sparsity_cnn = params.get("weight_sparsity_cnn", 1.0)
1249342
def build_from_queue(cls, input_queue, replay_size, batch_size):
    
    return cls(
        lambda: input_queue.dequeue_many(batch_size),
        replay_size,
        batch_size=batch_size)
250575
def create_batch(self, frame):
        
        frame_resize = mx.nd.array(cv2.resize(frame, (self.data_shape[0], self.data_shape[1])))
        #frame_resize = mx.img.imresize(frame, self.data_shape[0], self.data_shape[1], cv2.INTER_LINEAR)
        # Change dimensions from (w,h,channels) to (channels, w, h)
        frame_t = mx.nd.transpose(frame_resize, axes=(2,0,1))
        frame_norm = frame_t - self.mean_pixels_nd
        # Add dimension for batch, results in (1,channels,w,h)
        batch_frame = [mx.nd.expand_dims(frame_norm, axis=0)]
        batch_shape = [DataDesc('data', batch_frame[0].shape)]
        batch = DataBatch(data=batch_frame, provide_data=batch_shape)
        return batch
53973
def retrain(self):
        """"""
        folder = TrainData.from_folder(self.args.folder)
        train_data, test_data = folder.load(True, not self.args.no_validation)

        train_data = TrainData.merge(train_data, self.sampled_data)
        test_data = TrainData.merge(test_data, self.test)
        train_inputs, train_outputs = train_data
        print()
        try:
            self.listener.runner.model.fit(
                train_inputs, train_outputs, self.args.batch_size, self.epoch + self.args.epochs,
                validation_data=test_data, callbacks=self.callbacks, initial_epoch=self.epoch
            )
        finally:
            self.listener.runner.model.save(self.args.model)
405247
def compute_batch(self, duplicate_manager=None,context_manager=None):
        
        x, _ = self.acquisition.optimize(duplicate_manager=duplicate_manager)
        return x
773946
def learn(self, numEpochs, batchsize):
        """"""
        for epoch in range(numEpochs):
            print('epoch %d' % epoch)
            indexes = np.random.permutation(self.trainsize)
            for i in range(0, self.trainsize, batchsize):
                x = Variable(self.x_train[indexes[i: i + batchsize]])
                t = Variable(self.y_train[indexes[i: i + batchsize]])
                self.optimizer.update(self.model, x, t)
1262873
def stub_batch(cls, size, **kwargs):
        
        return [cls.stub(**kwargs) for _ in range(size)]
717718
def _create_job(self, mapping):
        """"""
        job_id = self.bulk.create_insert_job(mapping["sf_object"], contentType="CSV")
        self.logger.info("  Created bulk job {}".format(job_id))

        # Upload batches
        local_ids_for_batch = {}
        for batch_file, local_ids in self._get_batches(mapping):
            batch_id = self.bulk.post_batch(job_id, batch_file)
            local_ids_for_batch[batch_id] = local_ids
            self.logger.info("    Uploaded batch {}".format(batch_id))

        self.bulk.close_job(job_id)
        return job_id, local_ids_for_batch
1005362
def _load_general(data, targets):
    """"""
    for d_src, d_targets in zip(data, targets):
        if isinstance(d_targets, nd.NDArray):
            d_src.copyto(d_targets)
        else:
            assert d_targets[-1][0].stop == d_src.shape[0], \
                "Batch size miss match. Expected %d, got %d"%( \
                    d_targets[-1][0].stop, d_src.shape[0])
            for slice_idx, d_dst in d_targets:
                d_src[slice_idx].copyto(d_dst)
5629
def train():
    """"""
    trainer = gluon.Trainer(model.collect_params(), args.optimizer, {'learning_rate': args.lr})

    train_data_loader, val_data_loader, test_data_loader \
        = dataprocessor.make_dataloader(data_train, data_val, data_test, args)

    best_valid_bleu = 0.0
    for epoch_id in range(args.epochs):
        log_avg_loss = 0
        log_avg_gnorm = 0
        log_wc = 0
        log_start_time = time.time()
        for batch_id, (src_seq, tgt_seq, src_valid_length, tgt_valid_length)\
                in enumerate(train_data_loader):
            # logging.info(src_seq.context) Context suddenly becomes GPU.
            src_seq = src_seq.as_in_context(ctx)
            tgt_seq = tgt_seq.as_in_context(ctx)
            src_valid_length = src_valid_length.as_in_context(ctx)
            tgt_valid_length = tgt_valid_length.as_in_context(ctx)
            with mx.autograd.record():
                out, _ = model(src_seq, tgt_seq[:, :-1], src_valid_length, tgt_valid_length - 1)
                loss = loss_function(out, tgt_seq[:, 1:], tgt_valid_length - 1).mean()
                loss = loss * (tgt_seq.shape[1] - 1) / (tgt_valid_length - 1).mean()
                loss.backward()
            grads = [p.grad(ctx) for p in model.collect_params().values()]
            gnorm = gluon.utils.clip_global_norm(grads, args.clip)
            trainer.step(1)
            src_wc = src_valid_length.sum().asscalar()
            tgt_wc = (tgt_valid_length - 1).sum().asscalar()
            step_loss = loss.asscalar()
            log_avg_loss += step_loss
            log_avg_gnorm += gnorm
            log_wc += src_wc + tgt_wc
            if (batch_id + 1) % args.log_interval == 0:
                wps = log_wc / (time.time() - log_start_time)
                logging.info('[Epoch {} Batch {}/{}] loss={:.4f}, ppl={:.4f}, gnorm={:.4f}, '
                             'throughput={:.2f}K wps, wc={:.2f}K'
                             .format(epoch_id, batch_id + 1, len(train_data_loader),
                                     log_avg_loss / args.log_interval,
                                     np.exp(log_avg_loss / args.log_interval),
                                     log_avg_gnorm / args.log_interval,
                                     wps / 1000, log_wc / 1000))
                log_start_time = time.time()
                log_avg_loss = 0
                log_avg_gnorm = 0
                log_wc = 0
        valid_loss, valid_translation_out = evaluate(val_data_loader)
        valid_bleu_score, _, _, _, _ = compute_bleu([val_tgt_sentences], valid_translation_out)
        logging.info('[Epoch {}] valid Loss={:.4f}, valid ppl={:.4f}, valid bleu={:.2f}'
                     .format(epoch_id, valid_loss, np.exp(valid_loss), valid_bleu_score * 100))
        test_loss, test_translation_out = evaluate(test_data_loader)
        test_bleu_score, _, _, _, _ = compute_bleu([test_tgt_sentences], test_translation_out)
        logging.info('[Epoch {}] test Loss={:.4f}, test ppl={:.4f}, test bleu={:.2f}'
                     .format(epoch_id, test_loss, np.exp(test_loss), test_bleu_score * 100))
        dataprocessor.write_sentences(valid_translation_out,
                                      os.path.join(args.save_dir,
                                                   'epoch{:d}_valid_out.txt').format(epoch_id))
        dataprocessor.write_sentences(test_translation_out,
                                      os.path.join(args.save_dir,
                                                   'epoch{:d}_test_out.txt').format(epoch_id))
        if valid_bleu_score > best_valid_bleu:
            best_valid_bleu = valid_bleu_score
            save_path = os.path.join(args.save_dir, 'valid_best.params')
            logging.info('Save best parameters to {}'.format(save_path))
            model.save_parameters(save_path)
        if epoch_id + 1 >= (args.epochs * 2) // 3:
            new_lr = trainer.learning_rate * args.lr_update_factor
            logging.info('Learning rate change to {}'.format(new_lr))
            trainer.set_learning_rate(new_lr)
    if os.path.exists(os.path.join(args.save_dir, 'valid_best.params')):
        model.load_parameters(os.path.join(args.save_dir, 'valid_best.params'))
    valid_loss, valid_translation_out = evaluate(val_data_loader)
    valid_bleu_score, _, _, _, _ = compute_bleu([val_tgt_sentences], valid_translation_out)
    logging.info('Best model valid Loss={:.4f}, valid ppl={:.4f}, valid bleu={:.2f}'
                 .format(valid_loss, np.exp(valid_loss), valid_bleu_score * 100))
    test_loss, test_translation_out = evaluate(test_data_loader)
    test_bleu_score, _, _, _, _ = compute_bleu([test_tgt_sentences], test_translation_out)
    logging.info('Best model test Loss={:.4f}, test ppl={:.4f}, test bleu={:.2f}'
                 .format(test_loss, np.exp(test_loss), test_bleu_score * 100))
    dataprocessor.write_sentences(valid_translation_out,
                                  os.path.join(args.save_dir, 'best_valid_out.txt'))
    dataprocessor.write_sentences(test_translation_out,
                                  os.path.join(args.save_dir, 'best_test_out.txt'))
51375
def create_network(batch_size, update_freq):
    
    head = '%(asctime)-15s %(message)s'
    logging.basicConfig(level=logging.INFO, format=head)
    data = np.random.randint(1, 5, [1000, 2])

    #Test_Train data split
    n_train = int(data.shape[0] * 0.8)
    weights = np.array([1.0, 2.0])
    label = data.dot(weights)

    di = mx.io.NDArrayIter(data[:n_train, :], label[:n_train], batch_size=batch_size, shuffle=True, label_name='lin_reg_label')
    val_iter = mx.io.NDArrayIter(data[n_train:, :], label[n_train:], batch_size=batch_size)

    X = mx.sym.Variable('data')
    Y = mx.symbol.Variable('lin_reg_label')
    fully_connected_layer = mx.sym.FullyConnected(data=X, name='fc1', num_hidden=1)
    lro = mx.sym.LinearRegressionOutput(data=fully_connected_layer, label=Y, name="lro")

    mod = SVRGModule(
        symbol=lro,
        data_names=['data'],
        label_names=['lin_reg_label'], update_freq=update_freq, logger=logging)

    return di, val_iter, mod
188583
def _train_and_eval_batches(dataset, data_dir, input_name, num_devices):
  """"""
  (train_data, eval_data, features_info, keys) = train_and_eval_dataset(
      dataset, data_dir)
  input_names, target_names = keys[0], keys[1]
  train_batches = shuffle_and_batch_data(
      train_data, target_names, features_info, training=True,
      num_devices=num_devices)
  train_eval_batches = shuffle_and_batch_data(  # Data for eval-on-train.
      train_data, target_names, features_info, training=False,
      num_devices=num_devices)
  eval_batches = shuffle_and_batch_data(
      eval_data, target_names, features_info, training=False,
      num_devices=num_devices)
  input_name = input_name or input_names[0]
  input_shape = features_info[input_name].shape
  return (train_batches, train_eval_batches, eval_batches,
          input_name, list(input_shape))
84046
def train(train_dataset,
          eval_dataset,
          analysis_dir,
          output_dir,
          features,
          layer_sizes,
          max_steps=5000,
          num_epochs=None,
          train_batch_size=100,
          eval_batch_size=16,
          min_eval_frequency=100,
          learning_rate=0.01,
          epsilon=0.0005,
          job_name=None,
          cloud=None,
          ):
  """"""
  job = train_async(
      train_dataset=train_dataset,
      eval_dataset=eval_dataset,
      analysis_dir=analysis_dir,
      output_dir=output_dir,
      features=features,
      layer_sizes=layer_sizes,
      max_steps=max_steps,
      num_epochs=num_epochs,
      train_batch_size=train_batch_size,
      eval_batch_size=eval_batch_size,
      min_eval_frequency=min_eval_frequency,
      learning_rate=learning_rate,
      epsilon=epsilon,
      job_name=job_name,
      cloud=cloud,
  )
  job.wait()
  print('Training: ' + str(job.state))