Writes the median of the gradients to Tensorboard.
481945
def _write_median_gradient(self)->None:
        ""
        median_gradient = statistics.median(x.data.median() for x in self.gradients)
        self._add_gradient_scalar('median_gradient', scalar_value=median_gradient)
170584
def set_X_gradients(self, X, X_grad):
        """"""
        X.mean.gradient, X.variance.gradient = X_grad
80497
def _write_model_stats(self, iteration:int)->None:
        ""
        self.stats_writer.write(model=self.learn.model, iteration=iteration, tbwriter=self.tbwriter)
368058
def run():
  """"""
  step = tf.compat.v1.placeholder(tf.float32, shape=[])

  with tf.name_scope('loss'):
    # Specify 2 different loss values, each tagged differently.
    summary_lib.scalar('foo', tf.pow(0.9, step))
    summary_lib.scalar('bar', tf.pow(0.85, step + 2))

    # Log metric baz as well as upper and lower bounds for a margin chart.
    middle_baz_value = step + 4 * tf.random.uniform([]) - 2
    summary_lib.scalar('baz', middle_baz_value)
    summary_lib.scalar('baz_lower',
                       middle_baz_value - 6.42 - tf.random.uniform([]))
    summary_lib.scalar('baz_upper',
                       middle_baz_value + 6.42 + tf.random.uniform([]))

  with tf.name_scope('trigFunctions'):
    summary_lib.scalar('cosine', tf.cos(step))
    summary_lib.scalar('sine', tf.sin(step))
    summary_lib.scalar('tangent', tf.tan(step))

  merged_summary = tf.compat.v1.summary.merge_all()

  with tf.compat.v1.Session() as sess, tf.summary.FileWriter(LOGDIR) as writer:
    # We only need to specify the layout once (instead of per step).
    layout_summary = summary_lib.custom_scalar_pb(
        layout_pb2.Layout(category=[
            layout_pb2.Category(
                title='losses',
                chart=[
                    layout_pb2.Chart(
                        title='losses',
                        multiline=layout_pb2.MultilineChartContent(
                            tag=[r'loss(?!.*margin.*)'],)),
                    layout_pb2.Chart(
                        title='baz',
                        margin=layout_pb2.MarginChartContent(
                            series=[
                                layout_pb2.MarginChartContent.Series(
                                    value='loss/baz/scalar_summary',
                                    lower='loss/baz_lower/scalar_summary',
                                    upper='loss/baz_upper/scalar_summary'
                                ),
                            ],)),
                ]),
            layout_pb2.Category(
                title='trig functions',
                chart=[
                    layout_pb2.Chart(
                        title='wave trig functions',
                        multiline=layout_pb2.MultilineChartContent(
                            tag=[
                                r'trigFunctions/cosine', r'trigFunctions/sine'
                            ],)),
                    # The range of tangent is different. Give it its own chart.
                    layout_pb2.Chart(
                        title='tan',
                        multiline=layout_pb2.MultilineChartContent(
                            tag=[r'trigFunctions/tangent'],)),
                ],
                # This category we care less about. Make it initially closed.
                closed=True),
        ]))
    writer.add_summary(layout_summary)

    for i in xrange(42):
      summary = sess.run(merged_summary, feed_dict={step: i})
      writer.add_summary(summary, global_step=i)
731555
def make_gradients(dims=DEFAULT_DIMS):
    
    return np.meshgrid(
        np.linspace(0.0, 1.0, dims[0]),
        np.linspace(0.0, 1.0, dims[1])
    )
675628
def gradients_X_X2(self, dL_dK, X, X2):
        """"""
        return self._comp_grads(dL_dK, X, X2)[3:]
1004459
private int gradientUpdate(List<TensorBuilder> factors,
      List<TensorBuilder> unaryFactors, double stepSize) {
    // The best assignment, as computed from only the unary factors.
    int[] variableNums = new int[unaryFactors.size()];
    int[] variableSizes = new int[unaryFactors.size()];
    int[] variableValues = new int[unaryFactors.size()];
    locallyDecodeFactors(unaryFactors, variableNums, variableSizes, variableValues);

    // Identify where unary factors disagree with larger factors to compute the
    // subgradient.
    List<Integer> variableDisagreementList = Lists.newArrayList();
    List<Integer> factorDisagreementList = Lists.newArrayList();
    List<Integer> factorValueList = Lists.newArrayList();
    for (int i = 0; i < factors.size(); i++) {
      TensorBuilder factor = factors.get(i);
      int[] bestFactorValues = factor.keyNumToDimKey(factor.getLargestValues(1)[0]);
      int[] factorVariableNums = factor.getDimensionNumbers();

      for (int j = 0; j < factorVariableNums.length; j++) {
        int index = Arrays.binarySearch(variableNums, factorVariableNums[j]);
        if (bestFactorValues[j] != variableValues[index]) {
          // Factor and unary assignments disagree on variableNums[index].
          // Store the disagreement, used later to compute the subgradient.
          variableDisagreementList.add(index);
          factorDisagreementList.add(i);
          factorValueList.add(bestFactorValues[j]);

          /*
          System.out.println("Disagreement " + Arrays.toString(factorVariableNums) + "=" + Arrays.toString(bestFactorValues) + ", "
              + variableNums[index] + "=" + variableValues[index]);
              */
        }
      }
    }

    // Update the factor lagrange multipliers based on the disagreements.
    // For each disagreement, the subgradient update decreases the weight of
    // each factor's maximum weight assignment, and increases the weight of
    // the disagreeing factor's maximum weight assignment.
    for (int i = 0; i < variableDisagreementList.size(); i++) {
      int variableIndex = variableDisagreementList.get(i);
      int factorNum = factorDisagreementList.get(i);

      TensorBuilder factor = factors.get(factorNum);
      TensorBuilder unaryFactor = unaryFactors.get(variableIndex);

      int variableNum = variableNums[variableIndex];
      int variableSize = variableSizes[variableIndex];
      int bestFactorValue = factorValueList.get(i);
      int bestUnaryValue = variableValues[variableIndex];

      SparseTensor unaryGradient = SparseTensor.singleElement(new int[] { variableNum },
          new int[] { variableSize }, new int[] { bestUnaryValue }, 1);
      SparseTensor factorGradient = SparseTensor.singleElement(new int[] { variableNum },
          new int[] { variableSize }, new int[] { bestFactorValue }, 1);

      unaryFactor.incrementWithMultiplier(unaryGradient, -1.0 * stepSize);
      unaryFactor.incrementWithMultiplier(factorGradient, stepSize);
      factor.incrementWithMultiplier(unaryGradient, stepSize);
      factor.incrementWithMultiplier(factorGradient, -1.0 * stepSize);
    }

    return variableDisagreementList.size();
  }
1125860
private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
        ILossFunction lossFunction = layerConf().getLossFn();
        INDArray labels2d = getLabels2d(workspaceMgr, ArrayType.BP_WORKING_MEM);
        INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFn(), maskArray);
        org.deeplearning4j.nn.conf.ocnn.OCNNOutputLayer conf = ( org.deeplearning4j.nn.conf.ocnn.OCNNOutputLayer) conf().getLayer();


        if(conf.getLastEpochSinceRUpdated() == 0 && epochCount == 0) {
            INDArray currentR = doOutput(false,workspaceMgr);
            if(window == null) {
                window = Nd4j.createUninitializedDetached(conf.getWindowSize()).assign(0.0);
            }

            if(batchWindowSizeIndex < window.length() - currentR.length()) {
                window.put(new INDArrayIndex[]{NDArrayIndex.interval(batchWindowSizeIndex,batchWindowSizeIndex + currentR.length())},currentR);
            }
            else if(batchWindowSizeIndex < window.length()) {
                int windowIdx = (int) window.length() - batchWindowSizeIndex;
                window.put(new INDArrayIndex[]{NDArrayIndex.interval(window.length() - windowIdx,window.length())},currentR.get(NDArrayIndex.interval(0,windowIdx)));

            }

            batchWindowSizeIndex += currentR.length();
            conf.setLastEpochSinceRUpdated(epochCount);
        }
        else if(conf.getLastEpochSinceRUpdated()  != epochCount) {
            double percentile = window.percentileNumber(100.0 * conf.getNu()).doubleValue();
            getParam(R_KEY).putScalar(0,percentile);
            conf.setLastEpochSinceRUpdated(epochCount);
            batchWindowSizeIndex = 0;
        }
        else {
            //track a running average per minibatch per epoch
            //calculate the average r value quantl=ile
            //once the epoch changes

            INDArray currentR = doOutput(false,workspaceMgr);
            window.put(new INDArrayIndex[]{NDArrayIndex.interval(batchWindowSizeIndex,batchWindowSizeIndex + currentR.length())},currentR);
        }


        Gradient gradient = new DefaultGradient();
        INDArray vGradView = gradientViews.get(V_KEY);
        double oneDivNu = 1.0 / layerConf().getNu();
        INDArray xTimesV = input.mmul(getParam(V_KEY));
        INDArray derivW = layerConf().getActivationFn().getActivation(xTimesV.dup(),true).negi();
        derivW = derivW.muliColumnVector(delta).mean(0).muli(oneDivNu).addi(getParam(W_KEY));
        gradient.setGradientFor(W_KEY,gradientViews.get(W_KEY).assign(derivW));

        //dG -> sigmoid derivative

        INDArray firstVertDerivV =  layerConf().getActivationFn()
                .backprop(xTimesV.dup(),Nd4j.ones(input.dataType(), xTimesV.shape()))
                .getFirst().muliRowVector(getParam(W_KEY).neg());
        firstVertDerivV = firstVertDerivV.muliColumnVector(delta)
                        .reshape('f',input.size(0),1,layerConf().getHiddenSize());
        INDArray secondTermDerivV = input.reshape('f',
                input.size(0),getParam(V_KEY).size(0),1);

        long[]  shape = new long[firstVertDerivV.shape().length];
        for(int i = 0; i < firstVertDerivV.rank(); i++) {
            shape[i] = Math.max(firstVertDerivV.size(i),secondTermDerivV.size(i));
        }

        INDArray firstDerivVBroadcast = Nd4j.createUninitialized(input.dataType(), shape);

        INDArray mulResult = firstVertDerivV.broadcast(firstDerivVBroadcast);
        int[] bcDims = {0,1};
        Broadcast.mul(mulResult, secondTermDerivV, mulResult, bcDims);

        INDArray derivV = mulResult
                .mean(0).muli(oneDivNu).addi(getParam(V_KEY));
        gradient.setGradientFor(V_KEY,vGradView.assign(derivV));



        INDArray derivR = Nd4j.scalar(delta.meanNumber()).muli(oneDivNu).addi(-1);
        gradient.setGradientFor(R_KEY,gradientViews.get(R_KEY).assign(derivR));
        clearNoiseWeightParams();

        delta = backpropDropOutIfPresent(delta);
        return new Pair<>(gradient, delta);
    }
277707
def _output(cls, fluents: Sequence[FluentPair]) -> Sequence[tf.Tensor]:
        ''''''
        return tuple(cls._dtype(t) for t in cls._tensors(fluents))
740823
private Pair<Gradient, INDArray> getGradientsAndDelta(INDArray preOut, LayerWorkspaceMgr workspaceMgr) {
        ILossFunction lossFunction = layerConf().getLossFn();
        INDArray labels2d = getLabels2d(workspaceMgr, ArrayType.BP_WORKING_MEM);
        if (labels2d.size(1) != preOut.size(1)) {
            throw new DL4JInvalidInputException(
                            "Labels array numColumns (size(1) = " + labels2d.size(1) + ") does not match output layer"
                                            + " number of outputs (nOut = " + preOut.size(1) + ") " + layerId());
        }

        INDArray delta = lossFunction.computeGradient(labels2d, preOut, layerConf().getActivationFn(), maskArray);

        Gradient gradient = new DefaultGradient();

        INDArray weightGradView = gradientViews.get(CenterLossParamInitializer.WEIGHT_KEY);
        INDArray biasGradView = gradientViews.get(CenterLossParamInitializer.BIAS_KEY);
        INDArray centersGradView = gradientViews.get(CenterLossParamInitializer.CENTER_KEY);

        // centers delta
        double alpha = layerConf().getAlpha();

        INDArray centers = params.get(CenterLossParamInitializer.CENTER_KEY);
        INDArray l = labels.castTo(centers.dataType()); //Ensure correct dtype (same as params); no-op if already correct dtype
        INDArray centersForExamples = l.mmul(centers);
        INDArray diff = centersForExamples.sub(input).muli(alpha);
        INDArray numerator = l.transpose().mmul(diff);
        INDArray denominator = l.sum(0).reshape(l.size(1), 1).addi(1.0);

        INDArray deltaC;
        if (layerConf().getGradientCheck()) {
            double lambda = layerConf().getLambda();
            //For gradient checks: need to multiply dLc/dcj by lambda to get dL/dcj
            deltaC = numerator.muli(lambda);
        } else {
            deltaC = numerator.diviColumnVector(denominator);
        }
        centersGradView.assign(deltaC);



        // other standard calculations
        Nd4j.gemm(input, delta, weightGradView, true, false, 1.0, 0.0); //Equivalent to:  weightGradView.assign(input.transpose().mmul(delta));
        delta.sum(biasGradView, 0); //biasGradView is initialized/zeroed first in sum op

        gradient.gradientForVariable().put(CenterLossParamInitializer.WEIGHT_KEY, weightGradView);
        gradient.gradientForVariable().put(CenterLossParamInitializer.BIAS_KEY, biasGradView);
        gradient.gradientForVariable().put(CenterLossParamInitializer.CENTER_KEY, centersGradView);

        return new Pair<>(gradient, delta);
    }
282257
def write(self, learn:Learner, trn_batch:Tuple, val_batch:Tuple, iteration:int, tbwriter:SummaryWriter)->None:
        ""
        self._write_for_dstype(learn=learn, batch=val_batch, iteration=iteration, tbwriter=tbwriter, ds_type=DatasetType.Valid)
        self._write_for_dstype(learn=learn, batch=trn_batch, iteration=iteration, tbwriter=tbwriter, ds_type=DatasetType.Train)
966486
def update_gradients(self, grads):
        
        self.sigma2.gradient = grads[0]
        self.v.gradient = grads[1]
446359
def median(array):
    
    n = len(array)

    if n < 1:
        return 0
    elif n == 1:
        return array[0]

    sorted_vals = sorted(array)
    midpoint = int(n / 2)
    if n % 2 == 1:
        return sorted_vals[midpoint]
    else:
        return (sorted_vals[midpoint - 1] + sorted_vals[midpoint]) / 2.0
307308
def _summary(self, name, tensor):
    
    if tensor.shape.ndims == 0:
      return tf.summary.scalar(name, tensor)
    else:
      return tf.summary.histogram(name, tensor)
524166
def median(data):
    
    data = sorted(data)
    n = len(data)
    if n == 0:
        raise StatisticsError("no median for empty data")
    if n % 2 == 1:
        return data[n // 2]
    else:
        i = n // 2
        return (data[i - 1] + data[i]) / 2
1213509
def _write_model_stats(self, iteration:int)->None:
        ""
        # We don't want to write stats when model is not iterated on and hence has zeroed out gradients
        gen_mode = self.learn.gan_trainer.gen_mode
        if gen_mode and not self.gen_stats_updated: self._write_gen_model_stats(iteration=iteration)
        if not gen_mode and not self.crit_stats_updated: self._write_critic_model_stats(iteration=iteration)
371473
def summarize_tensors(tensor_dict, tag=None):
  
  if tag is None:
    tag = "tensors/"

  for t_name in list(tensor_dict):
    t = tensor_dict[t_name]
    tf.summary.histogram(tag + t_name, t)