Set weight decay and step optimizer.
480419
def step(self)->None:
        ""
        # weight decay outside of optimizer step (AdamW)
        if self.true_wd:
            for lr,wd,pg1,pg2 in zip(self._lr,self._wd,self.opt.param_groups[::2],self.opt.param_groups[1::2]):
                for p in pg1['params']: p.data.mul_(1 - wd*lr)
                if self.bn_wd:
                    for p in pg2['params']: p.data.mul_(1 - wd*lr)
            self.set_val('weight_decay', listify(0, self._wd))
        self.opt.step()
123141
def constant_cost_withGradients(x):
    
    return np.ones(x.shape[0])[:,None], np.zeros(x.shape)
1244593
def _optimize(self, period, tmpid, use_gradient=True):
        """"""
        theta_0 = [self.y.min(), self.y.max() - self.y.min(), 0]
        result = minimize(self._chi2, theta_0, jac=bool(use_gradient),
                          bounds=[(None, None), (0, None), (None, None)],
                          args=(period, tmpid, use_gradient))
        return result.x
903880
def ustep(self):
        """"""

        self.U += self.rsdl_r(self.AX, self.Y)
1143169
def on_step_end(self, **kwargs:Any)->None:
        ""
        #Zeros the gradients of the model since the optimizer is disconnected.
        self.learn.model.zero_grad()
        #Update the params from master to model.
        master2model(self.model_params, self.master_params, self.flat_master)
313326
function regularizeL2(weights, gradientCount, stepSize, regParam) {
  const [gradient, count] = gradientCount;

  for (let i = 0; i < gradient.length; i++) {
    let grad = (gradient[i] || 0) / count;
    weights[i] = weights[i] || 0;
    weights[i] -= stepSize * (grad + regParam * weights[i]);
  }
}
404117
def inverse_exp_decay(max_step, min_value=0.01, step=None):
  """"""
  inv_base = tf.exp(tf.log(min_value) / float(max_step))
  if step is None:
    step = tf.train.get_global_step()
  if step is None:
    return 1.0
  step = to_float(step)
  return inv_base**tf.maximum(float(max_step) - step, 0.0)
1129710
def optimize(self, G, params0=None, n_times=10, verbose=False, vmax=5, perturb=1e-3, factr=1e7):
        
        # set params0 from null if params0 is None
        if params0 is None:
            if self.null is None:
                if verbose:     print(".. fitting null model")
                self.fitNull()
            if self.bgRE:
                params0 = sp.concatenate([self.null['params0_g'], self.null['params0_n']])
            else:
                params0 = self.null['params0_n']
            params_was_None = True
        else:
            params_was_None = False
        G *= sp.sqrt(self.N/(G**2).sum())
        self._gp.covar.G = G
        start = TIME.time()
        for i in range(n_times):
            if params_was_None:
                n_params = self.Cr.getNumberParams()
                _params0 = {'covar': sp.concatenate([1e-3*sp.randn(n_params), params0])}
            else:
                _params0 = {'covar': params0}
            self._gp.setParams(_params0)
            conv, info = self._gp.optimize(factr=factr, verbose=verbose)
            conv *= self.Cr.K().diagonal().max()<vmax
            conv *= self.getLMLgrad() < 0.1
            if conv or not params_was_None: break
        self.infoOpt = info
        if not conv:
            warnings.warn("not converged")
        # return value
        RV = {}
        if self.P>1:
            RV['Cr']  = self.Cr.K()
            if self.bgRE: RV['Cg']  = self.Cg.K()
            RV['Cn']  = self.Cn.K()
        RV['time']  = sp.array([TIME.time()-start])
        RV['params0'] = _params0
        RV['nit'] = sp.array([info['nit']])
        RV['funcalls'] = sp.array([info['funcalls']])
        RV['var']    = self.getVariances()
        RV['conv']  = sp.array([conv])
        RV['NLLAlt']  = sp.array([self.getNLLAlt()])
        RV['LLR']    = sp.array([self.getLLR()])
        RV['LMLgrad'] = sp.array([self.getLMLgrad()])
        return RV
819709
def _global_step(hparams):
  """"""
  step = tf.to_float(tf.train.get_or_create_global_step())
  multiplier = hparams.optimizer_multistep_accumulate_steps
  if not multiplier:
    return step

  tf.logging.info("Dividing global step by %d for multi-step optimizer."
                  % multiplier)
  return step / tf.to_float(multiplier)
325566
def on_step_end(self, **kwargs):
        ""
        if not self.learn.gan_trainer.gen_mode: self.learn.opt.lr /= self.mult_lr
983403
def ystep(self):
        
        """

        self.Y = sp.prox_l1(self.AX - self.S + self.U, self.Wdf / self.rho)
644048
def robust_backtrack(self):
        

        self.L *= self.L_gamma_d
        maxiter = self.L_maxiter

        iterBTrack = 0
        linesearch = 1

        self.store_Yprev()
        while linesearch and iterBTrack < maxiter:

            t = float(1. + np.sqrt(1. + 4. * self.L * self.Tk)) / (2. * self.L)
            T = self.Tk + t
            y = (self.Tk * self.var_xprv() + t * self.ZZ) / T
            self.update_var_y(y)

            gradY = self.proximal_step()  # Given Y(f), L, this updates X(f)

            f = self.obfn_f(self.var_x())
            Dxy = self.eval_Dxy()
            Q = self.obfn_f(self.var_y()) + \
                self.eval_linear_approx(Dxy, gradY) + \
                (self.L / 2.) * np.linalg.norm(Dxy.flatten(), 2)**2

            if f <= Q:
                linesearch = 0
            else:
                self.L *= self.L_gamma_u

            iterBTrack += 1

        self.Tk = T
        self.ZZ += (t * self.L * (self.var_x() - self.var_y()))

        self.F = f
        self.Q = Q
        self.iterBTrack = iterBTrack
1367421
def solve
      set_parameters
      # Vectors holding the values used for calculating the weights:
      a = NArray.float(@n)
      b = NArray.float(@n)
      # Set an initial estimate for the probabilities of true segmentation:
      @n.times do |i|
        @weights_current[i] = @decisions[i, true].mean
      end
      # Proceed iteratively until we have converged to a local optimum:
      k = 0
      while k < max_iterations do
        # Copy weights:
        @weights_previous = @weights_current.dup
        # E-step: Estimation of the conditional expectation of the complete data log likelihood function.
        # Deriving the estimator for the unobserved true segmentation (T).
        @n.times do |i|
          voxel_decisions = @decisions[i, true]
          # Find the rater-indices for this voxel where the raters' decisions equals 1 and 0:
          positive_indices, negative_indices = (voxel_decisions.eq 1).where2
          # Determine ai:
          # Multiply by corresponding sensitivity (or 1 - sensitivity):
          a_decision1_factor = (positive_indices.length == 0 ? 1 : @p[positive_indices].prod)
          a_decision0_factor = (negative_indices.length == 0 ? 1 : (1 - @p[negative_indices]).prod)
          a[i] = @weights_previous[i] * a_decision1_factor * a_decision0_factor
          # Determine bi:
          # Multiply by corresponding specificity (or 1 - specificity):
          b_decision0_factor = (negative_indices.length == 0 ? 1 : @q[negative_indices].prod)
          b_decision1_factor = (positive_indices.length == 0 ? 1 : (1 - @q[positive_indices]).prod)
          b[i] = @weights_previous[i] * b_decision0_factor * b_decision1_factor
          # Determine Wi: (take care not to divide by zero)
          if a[i] > 0 or b[i] > 0
            @weights_current[i] = a[i] / (a[i] + b[i])
          else
            @weights_current[i] = 0
          end
        end
        # M-step: Estimation of the performance parameters by maximization.
        # Finding the values of the expert performance level parameters that maximize the conditional expectation
        # of the complete data log likelihood function (phi - p,q).
        @r.times do |j|
          voxel_decisions = @decisions[true, j]
          # Find the voxel-indices for this rater where the rater's decisions equals 1 and 0:
          positive_indices, negative_indices = (voxel_decisions.eq 1).where2
          # Determine sensitivity:
          # Sum the weights for the indices where the rater's decision equals 1:
          sum_positive = (positive_indices.length == 0 ? 0 : @weights_current[positive_indices].sum)
          @p[j] = sum_positive / @weights_current.sum
          # Determine specificity:
          # Sum the weights for the indices where the rater's decision equals 0:
          sum_negative = (negative_indices.length == 0 ? 0 : (1 - @weights_current[negative_indices]).sum)
          @q[j] = sum_negative / (1 - @weights_current).sum
        end
        # Bump our iteration index:
        k += 1
        # Abort if we have reached the local optimum: (there is no change in the sum of weights)
        if @weights_current.sum - @weights_previous.sum == 0
#puts "Iteration aborted as optimum solution was found!" if @verbose
#logger.info("Iteration aborted as optimum solution was found!")
          break
        end
      end
      # Set the true segmentation:
      @true_segmentation_vector = @weights_current.round
      # Set the weights attribute:
      @weights = @weights_current
      # As this vector doesn't make much sense to the user, it must be converted to a volume. If volume reduction has
      # previously been performed, this must be taken into account when transforming it to a volume:
      construct_segmentation_volume
      # Construct a BinVolume instance for the true segmentation and add it as a master volume to the BinMatcher instance.
      update_bin_matcher
      # Set the phi variable:
      @phi[0, true] = @p
      @phi[1, true] = @q
    end
450638
def reset_weights(self):
        
        init.uniform_(self.linear.weight, -3e-3, 3e-3)
        init.zeros_(self.linear.bias)
175506
def init_weights(self):
        """"""

        self.W = np.random.randn(self.n_neurons, self.n_inputs) * np.sqrt(2 / self.n_inputs)
        self.b = np.zeros((self.n_neurons, 1))
856584
def main(dimension, iterations):
    
    objective_function = minimize(functions.sphere)
    stopping_condition = max_iterations(iterations)
    (solution, metrics) = optimize(objective_function=objective_function,
                                   domain=Domain(-5.12, 5.12, dimension),
                                   stopping_condition=stopping_condition,
                                   parameters={'seed': 3758117674,
                                               'rho': 1.0, 'e_s': 15, 'e_f': 5},
                                   velocity_update=gc_velocity_update,
                                   parameter_update=update_rho,
                                   measurements=[fitness_measurement])
    return solution
105319
def updatePlaceWeights(self):
    
    self.weightsPI += np.outer(self.activationsI - self.boostTarget,
                               self.activationsP)*self.dt*\
                               self.learnFactorP*self.learningRate
    self.weightsPEL += np.outer(self.activationsEL - self.boostTarget,
                               self.activationsP)*self.dt*\
                               self.learnFactorP*self.learningRate
    self.weightsPER += np.outer(self.activationsER - self.boostTarget,
                               self.activationsP)*self.dt*\
                               self.learnFactorP*self.learningRate

    np.minimum(self.weightsPI, 1, self.weightsPI)
    np.minimum(self.weightsPEL, 1, self.weightsPEL)
    np.minimum(self.weightsPER, 1, self.weightsPER)