Training process
974496
def train():
    """"""
    start_pipeline_time = time.time()

    # Training/Testing
    best_valid_acc = 0
    stop_early = 0
    for epoch in range(args.epochs):
        # Epoch training stats
        start_epoch_time = time.time()
        epoch_L = 0.0
        epoch_sent_num = 0
        epoch_wc = 0
        # Log interval training stats
        start_log_interval_time = time.time()
        log_interval_wc = 0
        log_interval_sent_num = 0
        log_interval_L = 0.0

        for i, ((data, valid_length), label) in enumerate(train_dataloader):
            data = mx.nd.transpose(data.as_in_context(context))
            label = label.as_in_context(context)
            valid_length = valid_length.as_in_context(context).astype(np.float32)
            wc = valid_length.sum().asscalar()
            log_interval_wc += wc
            epoch_wc += wc
            log_interval_sent_num += data.shape[1]
            epoch_sent_num += data.shape[1]
            with autograd.record():
                output = net(data, valid_length)
                L = loss(output, label).mean()
            L.backward()
            # Clip gradient
            if args.clip is not None:
                grads = [p.grad(context) for p in net.collect_params().values()]
                gluon.utils.clip_global_norm(grads, args.clip)
            # Update parameter
            trainer.step(1)
            log_interval_L += L.asscalar()
            epoch_L += L.asscalar()
            if (i + 1) % args.log_interval == 0:
                print('[Epoch %d Batch %d/%d] avg loss %g, throughput %gK wps' % (
                    epoch, i + 1, len(train_dataloader),
                    log_interval_L / log_interval_sent_num,
                    log_interval_wc / 1000 / (time.time() - start_log_interval_time)))
                # Clear log interval training stats
                start_log_interval_time = time.time()
                log_interval_wc = 0
                log_interval_sent_num = 0
                log_interval_L = 0
        end_epoch_time = time.time()
        valid_avg_L, valid_acc = evaluate(valid_dataloader)
        test_avg_L, test_acc = evaluate(test_dataloader)
        print('[Epoch %d] train avg loss %g, '
              'valid acc %.4f, valid avg loss %g, '
              'test acc %.4f, test avg loss %g, throughput %gK wps' % (
                  epoch, epoch_L / epoch_sent_num,
                  valid_acc, valid_avg_L, test_acc, test_avg_L,
                  epoch_wc / 1000 / (end_epoch_time - start_epoch_time)))

        if valid_acc < best_valid_acc:
            print('No Improvement.')
            stop_early += 1
            if stop_early == 3:
                break
        else:
            # Reset stop_early if the validation loss finds a new low value
            print('Observed Improvement.')
            stop_early = 0
            net.save_parameters(args.save_prefix + '_{:04d}.params'.format(epoch))
            best_valid_acc = valid_acc

    net.load_parameters(glob.glob(args.save_prefix+'_*.params')[-1], context)
    valid_avg_L, valid_acc = evaluate(valid_dataloader)
    test_avg_L, test_acc = evaluate(test_dataloader)
    print('Best validation loss %g, validation acc %.4f'%(valid_avg_L, valid_acc))
    print('Best test loss %g, test acc %.4f'%(test_avg_L, test_acc))
    print('Total time cost %.2fs'%(time.time()-start_pipeline_time))
425949
def run_step(self):
        
        if not hasattr(self, 'train_op'):
            raise NotImplementedError(
                "Please either set `Trainer.train_op` or provide an implementation "
                "of Trainer.run_step()!")
        self.hooked_sess.run(self.train_op)
321798
def on_batch_begin(self, train, **kwargs):
        ""
        if not self.learn.gan_trainer.gen_mode and train: self.learn.opt.lr *= self.mult_lr
291809
def train(self, training_set, iterations=500):
        """"""
        if len(training_set) > 2:
            self.__X = np.matrix([example[0] for example in training_set])
            if self.__num_labels == 1:
                self.__y = np.matrix([example[1] for example in training_set]).reshape((-1, 1))
            else:
                eye = np.eye(self.__num_labels)
                self.__y = np.matrix([eye[example[1]] for example in training_set])
        else:
            self.__X = np.matrix(training_set[0])
            if self.__num_labels == 1:
                self.__y = np.matrix(training_set[1]).reshape((-1, 1))
            else:
                eye = np.eye(self.__num_labels)
                self.__y = np.matrix([eye[index] for sublist in training_set[1] for index in sublist])
        self.__m = self.__X.shape[0]
        self.__input_layer_size = self.__X.shape[1]
        self.__sizes = [self.__input_layer_size]
        self.__sizes.extend(self.__hidden_layers)
        self.__sizes.append(self.__num_labels)
        initial_theta = []
        for count in range(len(self.__sizes) - 1):
            epsilon = np.sqrt(6) / np.sqrt(self.__sizes[count]+self.__sizes[count+1])
            initial_theta.append(np.random.rand(self.__sizes[count+1],self.__sizes[count]+1)*2*epsilon-epsilon)
        initial_theta = self.__unroll(initial_theta)
        self.__thetas = self.__roll(fmin_bfgs(self.__cost_function, initial_theta, fprime=self.__cost_grad_function, maxiter=iterations))
552361
def train():
    """"""
    log.info('Loader Train data...')
    if version_2:
        train_data = SQuAD('train', version='2.0')
    else:
        train_data = SQuAD('train', version='1.1')
    log.info('Number of records in Train data:{}'.format(len(train_data)))

    train_data_transform, _ = preprocess_dataset(
        train_data, SQuADTransform(
            berttoken,
            max_seq_length=max_seq_length,
            doc_stride=doc_stride,
            max_query_length=max_query_length,
            is_pad=True,
            is_training=True))
    log.info('The number of examples after preprocessing:{}'.format(
        len(train_data_transform)))

    train_dataloader = mx.gluon.data.DataLoader(
        train_data_transform, batchify_fn=batchify_fn,
        batch_size=batch_size, num_workers=4, shuffle=True)

    log.info('Start Training')

    optimizer_params = {'learning_rate': lr}
    try:
        trainer = gluon.Trainer(net.collect_params(), optimizer,
                                optimizer_params, update_on_kvstore=False)
    except ValueError as e:
        print(e)
        warnings.warn('AdamW optimizer is not found. Please consider upgrading to '
                      'mxnet>=1.5.0. Now the original Adam optimizer is used instead.')
        trainer = gluon.Trainer(net.collect_params(), 'adam',
                                optimizer_params, update_on_kvstore=False)

    num_train_examples = len(train_data_transform)
    step_size = batch_size * accumulate if accumulate else batch_size
    num_train_steps = int(num_train_examples / step_size * epochs)
    num_warmup_steps = int(num_train_steps * warmup_ratio)
    step_num = 0

    def set_new_lr(step_num, batch_id):
        """set new learning rate"""
        # set grad to zero for gradient accumulation
        if accumulate:
            if batch_id % accumulate == 0:
                net.collect_params().zero_grad()
                step_num += 1
        else:
            step_num += 1
        # learning rate schedule
        # Notice that this learning rate scheduler is adapted from traditional linear learning
        # rate scheduler where step_num >= num_warmup_steps, new_lr = 1 - step_num/num_train_steps
        if step_num < num_warmup_steps:
            new_lr = lr * step_num / num_warmup_steps
        else:
            offset = (step_num - num_warmup_steps) * lr / \
                     (num_train_steps - num_warmup_steps)
            new_lr = lr - offset
        trainer.set_learning_rate(new_lr)
        return step_num

    # Do not apply weight decay on LayerNorm and bias terms
    for _, v in net.collect_params('.*beta|.*gamma|.*bias').items():
        v.wd_mult = 0.0
    # Collect differentiable parameters
    params = [p for p in net.collect_params().values()
              if p.grad_req != 'null']
    # Set grad_req if gradient accumulation is required
    if accumulate:
        for p in params:
            p.grad_req = 'add'

    epoch_tic = time.time()
    total_num = 0
    log_num = 0
    for epoch_id in range(epochs):
        step_loss = 0.0
        tic = time.time()
        for batch_id, data in enumerate(train_dataloader):
            # set new lr
            step_num = set_new_lr(step_num, batch_id)
            # forward and backward
            with mx.autograd.record():
                _, inputs, token_types, valid_length, start_label, end_label = data

                log_num += len(inputs)
                total_num += len(inputs)

                out = net(inputs.astype('float32').as_in_context(ctx),
                          token_types.astype('float32').as_in_context(ctx),
                          valid_length.astype('float32').as_in_context(ctx))

                ls = loss_function(out, [
                    start_label.astype('float32').as_in_context(ctx),
                    end_label.astype('float32').as_in_context(ctx)]).mean()

                if accumulate:
                    ls = ls / accumulate
            ls.backward()
            # update
            if not accumulate or (batch_id + 1) % accumulate == 0:
                trainer.allreduce_grads()
                nlp.utils.clip_grad_global_norm(params, 1)
                trainer.update(1)

            step_loss += ls.asscalar()

            if (batch_id + 1) % log_interval == 0:
                toc = time.time()
                log.info(
                    'Epoch: {}, Batch: {}/{}, Loss={:.4f}, lr={:.7f} Time cost={:.1f} Thoughput={:.2f} samples/s'  # pylint: disable=line-too-long
                        .format(epoch_id, batch_id, len(train_dataloader),
                                step_loss / log_interval,
                                trainer.learning_rate, toc - tic, log_num / (toc - tic)))
                tic = time.time()
                step_loss = 0.0
                log_num = 0
        epoch_toc = time.time()
        log.info('Epoch: {}, Time cost={:.2f} s, Thoughput={:.2f} samples/s'
                 .format(epoch_id, epoch_toc - epoch_tic,
                         len(train_dataloader) / (epoch_toc - epoch_tic)))

    net.save_parameters(os.path.join(output_dir, 'net.params'))
931036
def train(self, params, epoch):
    

    # Callback used to log information on every batch
    def log(model, batch_idx):
      if batch_idx % params["log_interval"] == 0:
        entropy = model.entropy()
        print("logging: {} learning iterations, entropy: {} / {}".format(
          model.getLearningIterations(), float(entropy), model.maxEntropy()))

        if params["create_plots"]:
          plotDutyCycles(model.dutyCycle,
                         self.resultsDir + "/figure_" + str(epoch) + "_" +
                         str(model.getLearningIterations()))


    # Adjust first epoch batch size to stabilize the dutycycles at the
    # beginning of the training
    loader = self.train_loader
    batches_in_epoch = params["batches_in_epoch"]
    if "first_epoch_batch_size" in params:
      if epoch == 0:
        batches_in_epoch = params.get("batches_in_first_epoch", batches_in_epoch)
        loader = torch.utils.data.DataLoader(self.train_loader.dataset,
                                             batch_size=params["first_epoch_batch_size"],
                                             sampler=self.train_loader.sampler)

    trainModel(model=self.model, loader=loader,
               optimizer=self.optimizer, device=self.device,
               batches_in_epoch=batches_in_epoch,
               batch_callback=log)

    self.model.postEpoch()
1260225
protected void initializeTraining(){
        if(!initializedTraining) {
            if(trainingConfig == null) {
                throw new ND4JIllegalStateException("Please specify a training config with setTrainingConfig");
            }
            //First: infer the variables to be optimized if required
            if(trainingConfig.getTrainableParams() == null || trainingConfig.getTrainableParams().size() == 0) {
                //Variable is trainable if it's not the output of some function
                //TODO also - should be floating point type
                List<String> trainVarList = new ArrayList<>();
                for(Variable var : variables.values()){
                    SDVariable v = var.getVariable();
                    String n = v.getVarName();
                    if(variables.get(n).getOutputOfOp() == null &&       //Is a leaf (not the output of a function)
                            !isPlaceHolder(n) &&                                //and not a placeholder
                            !variables.get(n).getVariable().isConstant() &&     //and not a constant
                            (trainingConfig.getDataSetFeatureMapping() == null || !trainingConfig.getDataSetFeatureMapping().contains(n))   &&  //and not an input (this really should be a placeholder, but we can't guarantee that...)
                            (trainingConfig.getDataSetLabelMapping() == null || !trainingConfig.getDataSetLabelMapping().contains(n))   &&      //and not a label (this really should be a placeholder, but we can't guarantee that...)
                            (trainingConfig.getDataSetFeatureMaskMapping() == null || !trainingConfig.getDataSetFeatureMaskMapping().contains(n))   &&  //and not a feature mask (this really should be a placeholder, but we can't guarantee that...)
                            (trainingConfig.getDataSetLabelMaskMapping() == null || !trainingConfig.getDataSetLabelMaskMapping().contains(n))){  //and not a label input (this really should be a placeholder, but we can't guarantee that...)
                        trainVarList.add(n);
                    }
                }

                trainingConfig.setTrainableParams(trainVarList);
                log.info("Inferred trainable variables: {}", trainVarList);
            }

            //Allocate updater state
            long numTrainableParams = 0;
            DataType dt = null;             //TODO support mixed precision variables - https://github.com/deeplearning4j/deeplearning4j/issues/6992
            for(String s : trainingConfig.getTrainableParams()) {
                SDVariable v = variables.get(s).getVariable();
                Preconditions.checkState(v != null, "No variable found for trainable parameter name \"%s\"", s);

                INDArray arr = v.getArr();
                Preconditions.checkState(arr != null, "No array found for trainable parameter \"%s\"", s);
                numTrainableParams += arr.length();
                if(dt == null)
                    dt = arr.dataType();
            }

            long updaterStateSize = trainingConfig.getUpdater().stateSize(numTrainableParams);

            if(updaterStateSize > 0) {
                try(MemoryWorkspace ws = Nd4j.getMemoryManager().scopeOutOfWorkspaces()) {
                    updaterState = Nd4j.createUninitialized(dt, 1, updaterStateSize);
                }
            }

            long viewSoFar = 0;
            updaterViews = new HashMap<>();
            updaterMap = new HashMap<>();
            for(String s : trainingConfig.getTrainableParams()) {
                long thisSize = trainingConfig.getUpdater().stateSize(variables.get(s).getVariable().getArr().length());
                INDArray view = (updaterStateSize == 0 || thisSize == 0 ? null :
                        updaterState.get(NDArrayIndex.interval(0, 1), NDArrayIndex.interval(viewSoFar, viewSoFar + thisSize)));

                updaterViews.put(s, view);
                updaterMap.put(s, trainingConfig.getUpdater().instantiate(view, true));
                viewSoFar += thisSize;
            }

            initializedTraining = true;
        }
    }
304003
def train(input_dir, batch_size, max_steps, output_dir, checkpoint, cloud_train_config):
    """"""

    import google.datalab.ml as ml
    if checkpoint is None:
      checkpoint = _util._DEFAULT_CHECKPOINT_GSURL
    staging_package_url = _util.repackage_to_staging(output_dir)
    job_args = {
      'input_dir': input_dir,
      'max_steps': max_steps,
      'batch_size': batch_size,
      'checkpoint': checkpoint
    }
    job_request = {
      'package_uris': [staging_package_url, _TF_GS_URL, _PROTOBUF_GS_URL],
      'python_module': 'mltoolbox.image.classification.task',
      'job_dir': output_dir,
      'args': job_args
    }
    job_request.update(dict(cloud_train_config._asdict()))
    job_id = 'image_classification_train_' + datetime.datetime.now().strftime('%y%m%d_%H%M%S')
    job = ml.Job.submit_training(job_request, job_id)
    if (_util.is_in_IPython()):
      import IPython
      log_url_query_strings = {
        'project': _util.default_project(),
        'resource': 'ml.googleapis.com/job_id/' + job.info['jobId']
      }
      log_url = 'https://console.developers.google.com/logs/viewer?' + \
          urllib.urlencode(log_url_query_strings)
      html = 'Job "%s" submitted.' % job.info['jobId']
      html += '<p>Click <a href="%s" target="_blank">here</a> to view cloud log. <br/>' % log_url
      IPython.display.display_html(html, raw=True)
    return job
1354002
protected function process_courses() {
        global $DB;
        $this->progress->start_progress(get_string('migratingcourses', 'tool_lpmigrate'), count($this->coursescompetencies));

        // Process each course.
        foreach ($this->coursescompetencies as $courseid => $competencyids) {
            $this->progress->increment_progress();

            $competenciestoremovefromcourse = array();
            $skipcompetencies = array();

            // First, add all the new competencies to the course.
            foreach ($competencyids as $key => $competencyid) {
                $this->coursecompetencyexpectedmigrations++;
                $mapto = isset($this->mappings[$competencyid]) ? $this->mappings[$competencyid] : false;

                // Skip the competencies that are not mapped.
                if ($mapto === false) {
                    $this->missingmappings[$competencyid] = true;

                    if ($this->removewhenmappingismissing) {
                        $competenciestoremovefromcourse[$competencyid] = true;
                    }

                    continue;
                }

                $transaction = $DB->start_delegated_transaction();
                try {
                    // Add the new competency to the course.
                    if (api::add_competency_to_course($courseid, $mapto)) {

                        // Find the added course competency.
                        $cc = course_competency::get_record(array('courseid' => $courseid, 'competencyid' => $mapto));

                        // Set the rule.
                        api::set_course_competency_ruleoutcome($cc, $this->coursescompetenciesoutcomes[$courseid][$competencyid]);

                        // Adapt the sortorder.
                        api::reorder_course_competency($courseid, $mapto, $competencyid);

                        $competenciestoremovefromcourse[$competencyid] = true;
                        $this->coursecompetencymigrations++;

                    } else {
                        // The competency was already in the course...
                        if ($this->removeoriginalwhenalreadypresent) {
                            $competenciestoremovefromcourse[$competencyid] = true;
                        } else {
                            $this->log_warning($courseid, $competencyid, null,
                                get_string('warningdestinationcoursecompetencyalreadyexists', 'tool_lpmigrate'));
                        }
                    }

                } catch (moodle_exception $e) {
                    // There was a major problem with this competency, we will ignore it entirely for the course.
                    $skipcompetencies[$competencyid] = true;

                    $this->log_error($courseid, $competencyid, null,
                        get_string('errorwhilemigratingcoursecompetencywithexception', 'tool_lpmigrate', $e->getMessage()));

                    try {
                        $transaction->rollback($e);
                    } catch (moodle_exception $e) {
                        // Catch the re-thrown exception.
                    }

                    continue;
                }
                $transaction->allow_commit();
            }

            // Then, convert the module competencies.
            if (!empty($this->modulecompetencies[$courseid])) {
                foreach ($this->modulecompetencies[$courseid] as $cmid => $competencyids) {
                    foreach ($competencyids as $competencyid) {
                        $this->modulecompetencyexpectedmigrations++;

                        // This mapped competency was not added to the course.
                        if (!empty($skipcompetencies[$competencyid])) {
                            continue;
                        }

                        $remove = true;
                        $mapto = isset($this->mappings[$competencyid]) ? $this->mappings[$competencyid] : false;

                        // We don't have mapping.
                        if ($mapto === false) {
                            if (!$this->removewhenmappingismissing) {
                                $remove = false;
                            }

                        } else {
                            // We have a mapping.
                            $transaction = $DB->start_delegated_transaction();
                            try {
                                // The competency was added successfully.
                                if (api::add_competency_to_course_module($cmid, $mapto)) {

                                    // Find the added module competency.
                                    $mc = course_module_competency::get_record(array('cmid' => $cmid, 'competencyid' => $mapto));

                                    // Set the competency rule.
                                    api::set_course_module_competency_ruleoutcome($mc,
                                        $this->modulecompetenciesoutcomes[$courseid][$cmid][$competencyid]);

                                    // Adapt the sortorder.
                                    api::reorder_course_module_competency($cmid, $mapto, $competencyid);

                                    $this->modulecompetencymigrations++;

                                } else {
                                    // The competency was already in the module.
                                    if (!$this->removeoriginalwhenalreadypresent) {
                                        $remove = false;
                                        $competencieswithissues[$competencyid] = true;
                                        $this->log_warning($courseid, $competencyid, $cmid,
                                            get_string('warningdestinationmodulecompetencyalreadyexists', 'tool_lpmigrate'));
                                    }
                                }

                            } catch (moodle_exception $e) {
                                // There was a major problem with this competency in this module.
                                $competencieswithissues[$competencyid] = true;
                                $message = get_string('errorwhilemigratingmodulecompetencywithexception', 'tool_lpmigrate',
                                    $e->getMessage());
                                $this->log_error($courseid, $competencyid, $cmid, $message);

                                try {
                                    $transaction->rollback($e);
                                } catch (moodle_exception $e) {
                                    // Catch the re-thrown exception.
                                }

                                continue;
                            }
                            $transaction->allow_commit();
                        }

                        try {
                            // Go away competency!
                            if ($remove && api::remove_competency_from_course_module($cmid, $competencyid)) {
                                $this->modulecompetencyremovals++;
                            }
                        } catch (moodle_exception $e) {
                            $competencieswithissues[$competencyid] = true;
                            $this->log_warning($courseid, $competencyid, $cmid,
                                get_string('warningcouldnotremovemodulecompetency', 'tool_lpmigrate'));
                        }
                    }
                }
            }

            // Finally, we remove the course competencies, but only for the 100% successful ones.
            foreach ($competenciestoremovefromcourse as $competencyid => $unused) {

                // Skip competencies with issues.
                if (isset($competencieswithissues[$competencyid])) {
                    continue;
                }

                try {
                    // Process the course competency.
                    api::remove_competency_from_course($courseid, $competencyid);
                    $this->coursecompetencyremovals++;
                } catch (moodle_exception $e) {
                    $this->log_warning($courseid, $competencyid, null,
                        get_string('warningcouldnotremovecoursecompetency', 'tool_lpmigrate'));
                }
            }
        }

        $this->progress->end_progress();
    }
1152170
def train():
    
    ntasgd = False
    best_val = float('Inf')
    start_train_time = time.time()
    parameters = model.collect_params()
    param_dict_avg = None
    t = 0
    avg_trigger = 0
    n = 5
    valid_losses = []
    for epoch in range(args.epochs):
        total_L = 0.0
        start_epoch_time = time.time()
        start_log_interval_time = time.time()
        hiddens = [model.begin_state(args.batch_size//len(context),
                                     func=mx.nd.zeros, ctx=ctx) for ctx in context]
        batch_i, i = 0, 0
        while i < len(train_data) - 1 - 1:
            bptt = args.bptt if mx.nd.random.uniform().asscalar() < 0.95 else args.bptt / 2
            seq_len = max(5, int(mx.nd.random.normal(bptt, 5).asscalar()))
            lr_batch_start = trainer.learning_rate
            trainer.set_learning_rate(lr_batch_start*seq_len/args.bptt)

            data, target = get_batch(train_data, i, seq_len=seq_len)
            data_list = gluon.utils.split_and_load(data, context, batch_axis=1, even_split=True)
            target_list = gluon.utils.split_and_load(target, context, batch_axis=1, even_split=True)
            hiddens = detach(hiddens)
            Ls = []
            with autograd.record():
                for j, (X, y, h) in enumerate(zip(data_list, target_list, hiddens)):
                    output, h, encoder_hs, dropped_encoder_hs = model(X, h)
                    l = joint_loss(output, y, encoder_hs, dropped_encoder_hs)
                    Ls.append(l / (len(context) * X.size))
                    hiddens[j] = h
            for L in Ls:
                L.backward()

            grads = [p.grad(d.context) for p in parameters.values() for d in data_list]
            gluon.utils.clip_global_norm(grads, args.clip)

            if args.ntasgd and ntasgd:
                if param_dict_avg is None:
                    param_dict_avg = {k.split(model._prefix)[1]: v.data(context[0]).copy()
                                      for k, v in parameters.items()}

            trainer.step(1)

            if args.ntasgd and ntasgd:
                gamma = 1.0 / max(1, epoch * (len(train_data) // args.bptt)
                                  + batch_i - avg_trigger + 2)
                for name, param_avg in param_dict_avg.items():
                    param_avg[:] += gamma * (parameters['{}{}'.format(model._prefix, name)]
                                             .data(context[0]) - param_avg)

            total_L += sum([mx.nd.sum(L).asscalar() for L in Ls])
            trainer.set_learning_rate(lr_batch_start)

            if batch_i % args.log_interval == 0 and batch_i > 0:
                cur_L = total_L / args.log_interval
                print('[Epoch %d Batch %d/%d] current loss %.2f, ppl %.2f, '
                      'throughput %.2f samples/s, lr %.2f'
                      % (epoch, batch_i, len(train_data) // args.bptt, cur_L, math.exp(cur_L),
                         args.batch_size * args.log_interval
                         / (time.time() - start_log_interval_time),
                         lr_batch_start * seq_len / args.bptt))
                total_L = 0.0
                start_log_interval_time = time.time()
            i += seq_len
            batch_i += 1

        mx.nd.waitall()

        print('[Epoch %d] throughput %.2f samples/s' % (
            epoch, (args.batch_size * len(train_data)) / (time.time() - start_epoch_time)))

        if args.ntasgd and ntasgd:
            mx.nd.save('{}.val.params'.format(args.save), param_dict_avg)
        else:
            model.save_parameters('{}.val.params'.format(args.save))
        val_L = evaluate(val_data, val_batch_size, '{}.val.params'.format(args.save), context[0])
        print('[Epoch %d] time cost %.2fs, valid loss %.2f, valid ppl %.2f，lr %.2f' % (
            epoch, time.time() - start_epoch_time, val_L, math.exp(val_L),
            trainer.learning_rate))

        if args.ntasgd and avg_trigger == 0:
            if t > n and val_L > min(valid_losses[-n:]):
                if param_dict_avg is None:
                    param_dict_avg = {k.split(model._prefix)[1]: v.data(context[0]).copy()
                                      for k, v in parameters.items()}
                else:
                    for k, v in parameters.items():
                        param_dict_avg[k.split(model._prefix)[1]] \
                            = v.data(context[0]).copy()
                avg_trigger = epoch * (len(train_data) // args.bptt) + len(train_data) // args.bptt
                print('Switching to NTASGD and avg_trigger is : %d' % avg_trigger)
                ntasgd = True
            valid_losses.append(val_L)
            t += 1

        if val_L < best_val:
            update_lr_epoch = 0
            best_val = val_L
            if args.ntasgd and ntasgd:
                mx.nd.save(args.save, param_dict_avg)
            else:
                model.save_parameters(args.save)
            test_L = evaluate(test_data, test_batch_size, args.save, context[0])
            print('[Epoch %d] test loss %.2f, test ppl %.2f'
                  % (epoch, test_L, math.exp(test_L)))
        else:
            update_lr_epoch += 1
            if update_lr_epoch % args.lr_update_interval == 0 and update_lr_epoch != 0:
                lr_scale = trainer.learning_rate * args.lr_update_factor
                print('Learning rate after interval update %f' % lr_scale)
                trainer.set_learning_rate(lr_scale)
                update_lr_epoch = 0

    print('Total training throughput %.2f samples/s'
          % ((args.batch_size * len(train_data) * args.epochs) / (time.time() - start_train_time)))
644233
def on_train_begin(self, **kwargs: Any) -> None:
        ""
        self.path.parent.mkdir(parents=True, exist_ok=True)      
        self.file = self.path.open('a') if self.append else self.path.open('w')
        self.file.write(','.join(self.learn.recorder.names[:(None if self.add_time else -1)]) + '\n')
351964
@Override
	public void initialize(Collection<Instance> trainingPoints)
	{
		Iterator<Instance> trgPtsIterator = trainingPoints.iterator();
		
		if(trgPtsIterator.hasNext() && this.numInstances == 0)
		{
			Instance inst = trgPtsIterator.next();
			this.buildForest(inst);
			this.trainOnInstance(inst);
		}
		
		while(trgPtsIterator.hasNext())
		{
			this.trainOnInstance((Instance)trgPtsIterator.next());			
		}
	}
1130867
def _create_training_directories():
    
    logger.info('Creating a new training folder under %s .' % base_dir)

    os.makedirs(model_dir)
    os.makedirs(input_config_dir)
    os.makedirs(output_data_dir)

    _write_json({}, hyperparameters_file_dir)
    _write_json({}, input_data_config_file_dir)

    host_name = socket.gethostname()

    resources_dict = {
        "current_host": host_name,
        "hosts": [host_name]
    }
    _write_json(resources_dict, resource_config_file_dir)
815009
def train(net, train_data, test_data):
    """"""
    start_pipeline_time = time.time()
    net, trainer = text_cnn.init(net, vocab, args.model_mode, context, args.lr)
    random.shuffle(train_data)
    sp = int(len(train_data)*0.9)
    train_dataloader = DataLoader(dataset=train_data[:sp],
                                  batch_size=args.batch_size,
                                  shuffle=True)
    val_dataloader = DataLoader(dataset=train_data[sp:],
                                batch_size=args.batch_size,
                                shuffle=False)
    test_dataloader = DataLoader(dataset=test_data,
                                 batch_size=args.batch_size,
                                 shuffle=False)
    # Training/Testing.
    best_val_acc = 0
    for epoch in range(args.epochs):
        # Epoch training stats.
        start_epoch_time = time.time()
        epoch_L = 0.0
        epoch_sent_num = 0
        epoch_wc = 0
        # Log interval training stats.
        start_log_interval_time = time.time()
        log_interval_wc = 0
        log_interval_sent_num = 0
        log_interval_L = 0.0
        for i, (data, label) in enumerate(train_dataloader):
            data = mx.nd.transpose(data.as_in_context(context))
            label = label.as_in_context(context)
            wc = max_len
            log_interval_wc += wc
            epoch_wc += wc
            log_interval_sent_num += data.shape[1]
            epoch_sent_num += data.shape[1]

            with autograd.record():
                output = net(data)
                L = loss(output, label).mean()
            L.backward()
            # Update parameter.
            trainer.step(1)
            log_interval_L += L.asscalar()
            epoch_L += L.asscalar()
            if (i + 1) % args.log_interval == 0:
                print('[Epoch %d Batch %d/%d] avg loss %g, throughput %gK wps' % (
                    epoch, i + 1, len(train_dataloader),
                    log_interval_L / log_interval_sent_num,
                    log_interval_wc / 1000 / (time.time() - start_log_interval_time)))
                # Clear log interval training stats.
                start_log_interval_time = time.time()
                log_interval_wc = 0
                log_interval_sent_num = 0
                log_interval_L = 0
        end_epoch_time = time.time()
        val_avg_L, val_acc = evaluate(net, val_dataloader)
        print('[Epoch %d] train avg loss %g, '
              'test acc %.4f, test avg loss %g, throughput %gK wps' % (
                  epoch, epoch_L / epoch_sent_num,
                  val_acc, val_avg_L,
                  epoch_wc / 1000 / (end_epoch_time - start_epoch_time)))

        if val_acc >= best_val_acc:
            print('Observed Improvement.')
            best_val_acc = val_acc
            test_avg_L, test_acc = evaluate(net, test_dataloader)

    print('Test loss %g, test acc %.4f'%(test_avg_L, test_acc))
    print('Total time cost %.2fs'%(time.time()-start_pipeline_time))
    return test_acc
718737
def train(self, data, target, **kwargs):
        
        non_predictors = [i.replace(" ", "_").lower() for i in list(set(data['team']))] + ["team", "next_year_wins"]
        self.column_names = [l for l in list(data.columns) if l not in non_predictors]
        results, folds = self.cross_validate(data, non_predictors, **kwargs)
        self.gather_results(results, folds, data)
459160
def run(self):
        
        device = self.model_config.torch_device()
        learner = Learner(device, self.model_factory.instantiate())

        # All callbacks useful for learning
        callbacks = self.gather_callbacks()

        # Metrics to track through this training
        metrics = learner.metrics()

        # Check if training was already started and potentially continue where we left off
        training_info, hidden_state = self.resume_training(learner, callbacks, metrics)

        # Prepare current training phase
        current_phase_idx = self._select_phase_left_bound(training_info.start_epoch_idx)
        current_phase = self.phases[current_phase_idx]
        local_idx = training_info.start_epoch_idx - self.ladder[current_phase_idx]

        current_phase.set_up_phase(training_info, learner.model, self.source)
        print(current_phase.banner())

        if training_info.start_epoch_idx > 0:
            current_phase.restore(training_info, local_idx, learner.model, hidden_state)

        training_info.on_train_begin()

        for global_epoch_idx in range(training_info.start_epoch_idx + 1, self.full_number_of_epochs + 1):
            iteration_phase_idx = self._select_phase_right_bound(global_epoch_idx-1)
            local_idx = global_epoch_idx - self.ladder[iteration_phase_idx]

            # Phase preparations
            while current_phase_idx != iteration_phase_idx:
                current_phase.tear_down_phase(training_info, learner.model)

                current_phase_idx += 1
                current_phase = self.phases[current_phase_idx]

                current_phase.set_up_phase(training_info, learner.model, self.source)
                print(current_phase.banner())

            # Create epoch info
            epoch_info = current_phase.epoch_info(training_info, global_epoch_idx, local_idx)

            # Execute learning
            current_phase.execute_epoch(epoch_info, learner)

            # Epoch checkpoint
            self.storage.checkpoint(epoch_info, learner.model)

        # Tear down the last phase
        if current_phase is not None:
            current_phase.tear_down_phase(training_info, learner.model)

        training_info.on_train_end()

        return training_info
1086510
private void trainOnChunk(Classifier classifierToTrain) {
		for (int num = 0; num < this.chunkSizeOption.getValue(); num++) {
			classifierToTrain.trainOnInstance(this.currentChunk.instance(num));
		}
	}