Scale gradients up by `self.loss_scale` to prevent underflow.
336146
def on_backward_begin(self, last_loss:Rank0Tensor, **kwargs:Any) -> Rank0Tensor:
        ""
        #To avoid gradient underflow, we scale the gradients
        ret_loss = last_loss * self.loss_scale
        return {'last_loss': ret_loss}
762670
def reset_scale_skew(self):
        
        chnames = self.fv.get_channel_names()
        for chname in chnames:
            chinfo = self.fv.get_channel(chname)

            scales = chinfo.fitsimage.get_scale_xy()
            fac = min(*scales)
            chinfo.fitsimage.scale_to(fac, fac)
395002
def backward(self, loss):
        """"""
        with mx.autograd.record():
            if isinstance(loss, (tuple, list)):
                ls = [l * self._scaler.loss_scale for l in loss]
            else:
                ls = loss * self._scaler.loss_scale
        mx.autograd.backward(ls)
933194
def update_gradients_full(self, dL_dK, X, X2=None, reset=True):
        
        self.variance.gradient = np.sum(self.K(X, X2)* dL_dK)/self.variance

        #now the lengthscale gradient(s)
        dL_dr = self.dK_dr_via_X(X, X2) * dL_dK
        if self.ARD:

            tmp = dL_dr*self._inv_dist(X, X2)
            if X2 is None: X2 = X
            if use_stationary_cython:
                self.lengthscale.gradient = self._lengthscale_grads_cython(tmp, X, X2)
            else:
                self.lengthscale.gradient = self._lengthscale_grads_pure(tmp, X, X2)
        else:
            r = self._scaled_dist(X, X2)
            self.lengthscale.gradient = -np.sum(dL_dr*r)/self.lengthscale
452828
private static void scaleDownFive(Slice decimal, int fiveScale, Slice result)
    {
        while (true) {
            int powerFive = Math.min(fiveScale, MAX_POWER_OF_FIVE_INT);
            fiveScale -= powerFive;

            int divisor = POWERS_OF_FIVES_INT[powerFive];
            divide(decimal, divisor, result);
            decimal = result;

            if (fiveScale == 0) {
                return;
            }
        }
    }
83591
def gradient_list(start, end, steps):
    """"""
    delta = [end[i] - start[i] for i in range(4)]
    return [bytearray(start[j] + (delta[j] * i) // steps for j in range(4))
            for i in range(steps + 1)]
1240731
def gradients_X(self, dL_dK, X, X2):
        """"""
        return self._comp_grads(dL_dK, X, X2)[3]
461305
def train(self, x):
        
        self.range = scale_continuous.train(x, self.range)
338382
private static BigDecimal roundedTenPower(int qsign, int raise, int scale, int preferredScale) {
        if (scale > preferredScale) {
            int diff = scale - preferredScale;
            if(diff < raise) {
                return scaledTenPow(raise - diff, qsign, preferredScale);
            } else {
                return valueOf(qsign,scale-raise);
            }
        } else {
            return scaledTenPow(raise, qsign, scale);
        }
    }
781183
def gradient(self):
        """"""
        try:
            op_to_return = self.functional.gradient
        except NotImplementedError:
            raise NotImplementedError(
                '`self.functional.gradient` is not implemented for '
                '`self.functional` {}'.format(self.functional))

        op_to_return = op_to_return - ConstantOperator(self.subgrad)
        return op_to_return
680060
def _gradient_penalty(self, real_samples, fake_samples, kwargs):
        
        import torch
        from torch.autograd import Variable, grad

        real_samples = real_samples.view(fake_samples.shape)

        subset_size = real_samples.shape[0]

        real_samples = real_samples[:subset_size]
        fake_samples = fake_samples[:subset_size]

        alpha = torch.rand(subset_size)
        if self.use_cuda:
            alpha = alpha.cuda()
        alpha = alpha.view((-1,) + ((1,) * (real_samples.dim() - 1)))

        interpolates = alpha * real_samples + ((1 - alpha) * fake_samples)
        interpolates = Variable(interpolates, requires_grad=True)
        if self.use_cuda:
            interpolates = interpolates.cuda()

        d_output = self.critic(interpolates, **kwargs)

        grad_ouputs = torch.ones(d_output.size())
        if self.use_cuda:
            grad_ouputs = grad_ouputs.cuda()

        gradients = grad(
            outputs=d_output,
            inputs=interpolates,
            grad_outputs=grad_ouputs,
            create_graph=True,
            retain_graph=True,
            only_inputs=True)[0]
        return ((gradients.norm(2, dim=1) - 1) ** 2).mean() * 10
313326
function regularizeL2(weights, gradientCount, stepSize, regParam) {
  const [gradient, count] = gradientCount;

  for (let i = 0; i < gradient.length; i++) {
    let grad = (gradient[i] || 0) / count;
    weights[i] = weights[i] || 0;
    weights[i] -= stepSize * (grad + regParam * weights[i]);
  }
}
1149865
def smooth_l1_loss(input, target, beta=1. / 9, size_average=True):
    
    n = torch.abs(input - target)
    cond = n < beta
    loss = torch.where(cond, 0.5 * n ** 2 / beta, n - 0.5 * beta)
    if size_average:
        return loss.mean()
    return loss.sum()
1105191
def average_gradient(data, *kwargs):
    
    return np.average(np.array(np.gradient(data))**2)
117578
def resetScale(self):
        """"""
        self.img.scale(1./self.imgScale[0], 1./self.imgScale[1])
        self.imgScale = (1.,1.)
38055
def dropout_no_scaling(x, keep_prob):
  
  if keep_prob == 1.0:
    return x
  mask = tf.less(tf.random_uniform(tf.shape(x)), keep_prob)
  return x * cast_like(mask, x)
689739
def _backprop_gradient_pure(dL, L):
    
    dL_dK = np.tril(dL).copy()
    N = L.shape[0]
    for k in range(N - 1, -1, -1):
        for j in range(k + 1, N):
            for i in range(j, N):
                dL_dK[i, k] -= dL_dK[i, j] * L[j, k]
                dL_dK[j, k] -= dL_dK[i, j] * L[i, k]
        for j in range(k + 1, N):
            dL_dK[j, k] /= L[k, k]
            dL_dK[k, k] -= L[j, k] * dL_dK[j, k]
        dL_dK[k, k] /= (2 * L[k, k])
    return dL_dK