Clip the gradient before the optimizer step.
429717
def on_backward_end(self, **kwargs):
        ""
        if self.clip: nn.utils.clip_grad_norm_(self.learn.model.parameters(), self.clip)
1312098
def rescale_gradients(model: Model, grad_norm: Optional[float] = None) -> Optional[float]:
    
    if grad_norm:
        parameters_to_clip = [p for p in model.parameters()
                              if p.grad is not None]
        return sparse_clip_norm(parameters_to_clip, grad_norm)
    return None
677338
private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
        ILossFunction lossFunction = layerConf().getLossFn();
        INDArray labels2d = getLabels2d(workspaceMgr, ArrayType.BP_WORKING_MEM);
        //INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFunction(), maskArray);
        INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFn(), maskArray);

        Gradient gradient = new DefaultGradient();

        INDArray weightGradView = gradientViews.get(DefaultParamInitializer.WEIGHT_KEY);
        Nd4j.gemm(input.castTo(weightGradView.dataType()), delta, weightGradView, true, false, 1.0, 0.0); //Equivalent to:  weightGradView.assign(input.transpose().mmul(delta));         //TODO can we avoid cast?
        gradient.gradientForVariable().put(DefaultParamInitializer.WEIGHT_KEY, weightGradView);

        if(hasBias()){
            INDArray biasGradView = gradientViews.get(DefaultParamInitializer.BIAS_KEY);
            delta.sum(biasGradView, 0); //biasGradView is initialized/zeroed first in sum op
            gradient.gradientForVariable().put(DefaultParamInitializer.BIAS_KEY, biasGradView);
        }

        delta = workspaceMgr.leverageTo(ArrayType.ACTIVATION_GRAD, delta);
        return new Pair<>(gradient, delta);
    }
12518
def vertical_gradient(self, x0, y0, x1, y1, start, end):
        """"""
        x0, y0, x1, y1 = self.rect_helper(x0, y0, x1, y1)
        grad = gradient_list(start, end, y1 - y0)
        for x in range(x0, x1 + 1):
            for y in range(y0, y1 + 1):
                self.point(x, y, grad[y - y0])
959707
def gen_gradient(args, resource, inflow, radius, loc, common=True):
    
    return "".join(["GRADIENT_RESOURCE ", str(resource), ":height=",
                    str(radius), ":plateau=", str(inflow), ":spread=",
                    str(radius-1), ":common=", str(int(common)),
                    ":updatestep=1000000:peakx=", str(loc[0]), ":peaky=",
                    str(loc[1]), ":plateau_inflow=", str(inflow), ":initial=",
                    str(inflow) + "\n"])
1285221
def __clip(val, minimum, maximum):
        
        if val is None or minimum is None or maximum is None:
            return None
        if val < minimum:
            return minimum
        if val > maximum:
            return maximum
        return val
735560
def limit_step(self, step):
        """"""
        if self.qmax is None:
            return step
        else:
            return np.clip(step, -self.qmax, self.qmax)
720558
def optimise_gradient_descent(x, a, c, tolerance=0.001):
    
    x_new = x
    x_old = x-1 # dummy value
    while np.abs(x_new - x_old).sum() > tolerance:
        x_old = x_new
        step_size = golden_section_search(lambda step: f(grad_desc_update(x_old, a, c, step), a, c), -1.0, 1.0)
        x_new = grad_desc_update(x_old, a, c, step_size)
    return x_new
944122
def step(self, loss, optimizer, scheduler, update=True):
        
        loss.backward()
        if update:
            if self.grad_clip != float('inf'):
                clip_grad_norm_(self.model.parameters(), self.grad_clip)
            scheduler.step()
            optimizer.step()
            self.model.zero_grad()
503679
def optimize(args):
    
    if args.cuda:
        ctx = mx.gpu(0)
    else:
        ctx = mx.cpu(0)
    # load the content and style target
    content_image = utils.tensor_load_rgbimage(args.content_image,ctx, size=args.content_size, keep_asp=True)
    content_image = utils.subtract_imagenet_mean_preprocess_batch(content_image)
    style_image = utils.tensor_load_rgbimage(args.style_image, ctx, size=args.style_size)
    style_image = utils.subtract_imagenet_mean_preprocess_batch(style_image)
    # load the pre-trained vgg-16 and extract features
    vgg = net.Vgg16()
    utils.init_vgg_params(vgg, 'models', ctx=ctx)
    # content feature
    f_xc_c = vgg(content_image)[1]
    # style feature
    features_style = vgg(style_image)
    gram_style = [net.gram_matrix(y) for y in features_style]
    # output
    output = Parameter('output', shape=content_image.shape)
    output.initialize(ctx=ctx)
    output.set_data(content_image)
    # optimizer
    trainer = gluon.Trainer([output], 'adam',
                            {'learning_rate': args.lr})
    mse_loss = gluon.loss.L2Loss()

    # optimizing the images
    for e in range(args.iters):
        utils.imagenet_clamp_batch(output.data(), 0, 255)
        # fix BN for pre-trained vgg
        with autograd.record():
            features_y = vgg(output.data())
            content_loss = 2 * args.content_weight * mse_loss(features_y[1], f_xc_c)
            style_loss = 0.
            for m in range(len(features_y)):
                gram_y = net.gram_matrix(features_y[m])
                gram_s = gram_style[m]
                style_loss = style_loss + 2 * args.style_weight * mse_loss(gram_y, gram_s)
            total_loss = content_loss + style_loss
            total_loss.backward()

        trainer.step(1)
        if (e + 1) % args.log_interval == 0:
            print('loss:{:.2f}'.format(total_loss.asnumpy()[0]))

    # save the image
    output = utils.add_imagenet_mean_batch(output.data())
    utils.tensor_save_bgrimage(output[0], args.output_image, args.cuda)
125156
def clip(self, lower=None, upper=None):
    '''
    
    '''
    df = self.export_df()
    df = df.clip(lower=lower, upper=upper)
    self.load_df(df)
210947
def prepare(self, f):
        """"""
        self.g = autograd.grad(f)
        self.h = autograd.hessian(f)
319001
def fit(self, n_iter, eta=0.01):
        """"""
        for _ in range(n_iter):
            gradient = self.compute_gradient()
            self.gradient_step(gradient, eta)
327560
private void minimizeWithGeometricConstraints() {
		extractEpipoles.setTensor(solutionN);
		extractEpipoles.extractEpipoles(e2,e3);

		// encode the parameters being optimized
		param[0] = e2.x; param[1] = e2.y; param[2] = e2.z;
		param[3] = e3.x; param[4] = e3.y; param[5] = e3.z;

		// adjust the error function for the current inputs
		errorFunction.init();

		// set up the optimization algorithm
		optimizer.setFunction(errorFunction,null);
		optimizer.initialize(param,gtol,ftol);

		// optimize until convergence or the maximum number of iterations
		UtilOptimize.process(optimizer, maxIterations);

		// get the results and compute the trifocal tensor
		double found[] = optimizer.getParameters();
		paramToEpipoles(found,e2,e3);

		enforce.process(e2,e3,A);
		enforce.extractSolution(solutionN);
	}
232860
function regularizeNone(weights, gradientCount) {
  const [gradient, count] = gradientCount;

  for (let i = 0; i < gradient.length; i++) {
    let grad = (gradient[i] || 0) / count;
    weights[i] = (weights[i] || 0) - grad;
  }
}
987131
def run_optimization(self):
    
    penalty_val = self.params['init_penalty']
    # Don't use smoothing initially - very inaccurate for large dimension
    self.smooth_on = False
    smooth_val = 0
    learning_rate_val = self.params['init_learning_rate']
    self.current_outer_step = 1


    while self.current_outer_step <= self.params['outer_num_steps']:
      tf.logging.info('Running outer step %d with penalty %f',
                      self.current_outer_step, penalty_val)
      # Running inner loop of optimization with current_smooth_val,
      # current_penalty as smoothness parameters and penalty respectively
      self.current_step = 0
      # Run first step with random eig initialization and large number of steps
      found_cert = self.run_one_step(
          self.dual_object.m_min_vec_estimate,
          self.params['large_eig_num_steps'], smooth_val, penalty_val, learning_rate_val)
      if found_cert:
        return True
      while self.current_step < self.params['inner_num_steps']:
        self.current_step = self.current_step + 1
        found_cert = self.run_one_step(self.dual_object.m_min_vec_estimate,
                                       self.params['small_eig_num_steps'],
                                       smooth_val, penalty_val,
                                       learning_rate_val)
        if found_cert:
          return True
      # Update penalty only if it looks like current objective is optimizes
      if self.current_total_objective < UPDATE_PARAM_CONSTANT:
        penalty_val = penalty_val * self.params['beta']
        learning_rate_val = learning_rate_val*self.params['learning_rate_decay']
      else:
        # To get more accurate gradient estimate
        self.params['small_eig_num_steps'] = (
            1.5 * self.params['small_eig_num_steps'])

      # If eigen values seem small enough, turn on smoothing
      # useful only when performing full eigen decomposition
      if np.abs(self.current_eig_val_estimate) < 0.01:
        smooth_val = self.params['smoothness_parameter']
      self.current_outer_step = self.current_outer_step + 1
    return False
781183
def gradient(self):
        """"""
        try:
            op_to_return = self.functional.gradient
        except NotImplementedError:
            raise NotImplementedError(
                '`self.functional.gradient` is not implemented for '
                '`self.functional` {}'.format(self.functional))

        op_to_return = op_to_return - ConstantOperator(self.subgrad)
        return op_to_return