r'''
    We introduce a function for logging a tensor variable's current state.
    It logs scalar values for the mean, standard deviation, minimum and maximum.
    Furthermore it logs a histogram of its state and (if given) of an optimization gradient.
438295
def log_variable(variable, gradient=None):
    
    '''
    name = variable.name.replace(':', '_')
    mean = tf.reduce_mean(variable)
    tf.summary.scalar(name='%s/mean'   % name, tensor=mean)
    tf.summary.scalar(name='%s/sttdev' % name, tensor=tf.sqrt(tf.reduce_mean(tf.square(variable - mean))))
    tf.summary.scalar(name='%s/max'    % name, tensor=tf.reduce_max(variable))
    tf.summary.scalar(name='%s/min'    % name, tensor=tf.reduce_min(variable))
    tf.summary.histogram(name=name, values=variable)
    if gradient is not None:
        if isinstance(gradient, tf.IndexedSlices):
            grad_values = gradient.values
        else:
            grad_values = gradient
        if grad_values is not None:
            tf.summary.histogram(name='%s/gradients' % name, values=grad_values)
860634
private double computeOutputError(double[] output, double[] gradient) {
        if (output.length != outputLayer.units) {
            throw new IllegalArgumentException(String.format("Invalid output vector size: %d, expected: %d", output.length, outputLayer.units));
        }

        double error = 0.0;
        for (int i = 0; i < outputLayer.units; i++) {
            double out = outputLayer.output[i];
            double g = output[i] - out;

            if (errorFunction == ErrorFunction.LEAST_MEAN_SQUARES) {
                error += 0.5 * g * g;
            } else if (errorFunction == ErrorFunction.CROSS_ENTROPY) {
                if (activationFunction == ActivationFunction.SOFTMAX) {
                    error -= output[i] * log(out);
                } else if (activationFunction == ActivationFunction.LOGISTIC_SIGMOID) {
                    // We have only one output neuron in this case.
                    error = -output[i] * log(out) - (1.0 - output[i]) * log(1.0 - out);
                }
            }

            if (errorFunction == ErrorFunction.LEAST_MEAN_SQUARES && activationFunction == ActivationFunction.LOGISTIC_SIGMOID) {
                g *= out * (1.0 - out);
            }

            gradient[i] = g;
        }

        return error;
    }
155102
def debug_tensor(tensor, name):
    
    logging.info(name)
    tensor = tensor.detach().float().cpu().numpy()
    logging.info(f'MIN: {tensor.min()} MAX: {tensor.max()} '
                 f'AVG: {tensor.mean()} STD: {tensor.std()} '
                 f'NAN: {np.isnan(tensor).sum()} INF: {np.isinf(tensor).sum()}')
829212
def gradient_helper(optimizer, loss, var_list=None):
    '''
    '''
    if var_list is None:
      var_list = tf.compat.v1.trainable_variables()

    grads_and_vars = optimizer.compute_gradients(loss, var_list=var_list)
    grads = [pair[0] for pair in grads_and_vars]

    return grads, optimizer.apply_gradients(grads_and_vars)
1312404
def _log_prior_gradients(self):
        """"""
        if self.priors.size == 0:
            return 0.
        x = self.param_array
        ret = np.zeros(x.size)
        #compute derivate of prior density
        [np.put(ret, ind, p.lnpdf_grad(x[ind])) for p, ind in self.priors.items()]
        #add in jacobian derivatives if transformed
        priored_indexes = np.hstack([i for p, i in self.priors.items()])
        for c,j in self.constraints.items():
            if not isinstance(c, Transformation):continue
            for jj in j:
                if jj in priored_indexes:
                    ret[jj] += c.log_jacobian_grad(x[jj])
        return ret
470228
def min_tens(tens, rmax=10, nswp=10, verb=True, smooth_fun=None):
    """"""
    if smooth_fun is None:
        smooth_fun = lambda p, lam: (math.pi / 2 - np.arctan(p - lam))
    d = tens.d
    Rx = [[]] * (d + 1)  # Python list for the interfaces
    Rx[0] = np.ones((1, 1))
    Rx[d] = np.ones((1, 1))
    Jy = [np.empty(0, dtype=np.int)] * (d + 1)
    ry = rmax * np.ones(d + 1, dtype=np.int)
    ry[0] = 1
    ry[d] = 1
    n = tens.n
    elements_seen = 0
    phi_left = [np.empty(0)] * (d + 1)
    phi_left[0] = np.array([1])
    phi_right = [np.empty(0)] * (d + 1)
    phi_right[d] = np.array([1])
    cores = tt.tensor.to_list(tens)

    # Fill initial multiindex J randomly.
    grid = [np.reshape(range(n[i]), (n[i], 1)) for i in xrange(d)]
    for i in xrange(d - 1):
        ry[i + 1] = min(ry[i + 1], n[i] * ry[i])
        ind = sorted(np.random.permutation(ry[i] * n[i])[0:ry[i + 1]])
        w1 = mkron(np.ones((n[i], 1), dtype=np.int), Jy[i])
        w2 = mkron(grid[i], np.ones((ry[i], 1), dtype=np.int))
        Jy[i + 1] = np.hstack((w1, w2))
        Jy[i + 1] = reshape(Jy[i + 1], (ry[i] * n[i], -1))
        Jy[i + 1] = Jy[i + 1][ind, :]
        phi_left[i + 1] = np.tensordot(phi_left[i], cores[i], 1)
        phi_left[i + 1] = reshape(phi_left[i + 1], (ry[i] * n[i], -1))
        phi_left[i + 1] = phi_left[i + 1][ind, :]
    swp = 0
    dirn = -1
    i = d - 1
    lm = float('Inf')
    while swp < nswp:
        # Right-to-left sweep
        # The idea: compute the current core; compute the function of it;
        # Shift locally or globally? Local shift would be the first try
        # Compute the current core

        if np.size(Jy[i]) == 0:
            w1 = np.zeros((ry[i] * n[i] * ry[i + 1], 0), dtype=np.int)
        else:
            w1 = mkron(np.ones((n[i] * ry[i + 1], 1), dtype=np.int), Jy[i])
        w2 = mkron(mkron(np.ones((ry[i + 1], 1), dtype=np.int),
                         grid[i]), np.ones((ry[i], 1), dtype=np.int))
        if np.size(Jy[i + 1]) == 0:
            w3 = np.zeros((ry[i] * n[i] * ry[i + 1], 0), dtype=np.int)
        else:
            w3 = mkron(Jy[i + 1], np.ones((ry[i] * n[i], 1), dtype=np.int))
        J = np.hstack((w1, w2, w3))

        phi_right[i] = np.tensordot(cores[i], phi_right[i + 1], 1)
        phi_right[i] = reshape(phi_right[i], (-1, n[i] * ry[i + 1]))

        cry = np.tensordot(
            phi_left[i], np.tensordot(
                cores[i], phi_right[
                    i + 1], 1), 1)
        elements_seen += cry.size
        cry = reshape(cry, (ry[i], n[i], ry[i + 1]))
        min_cur = np.min(cry.flatten("F"))
        ind_cur = np.argmin(cry.flatten("F"))
        if lm > min_cur:
            lm = min_cur
            x_full = J[ind_cur, :]
            val = tens[x_full]
            if verb:
                print('New record:', val, 'Point:', x_full, 'elements seen:', elements_seen)
        cry = smooth_fun(cry, lm)
        if dirn < 0 and i > 0:
            cry = reshape(cry, (ry[i], n[i] * ry[i + 1]))
            cry = cry.T
            #q, r = np.linalg.qr(cry)
            u, s, v = mysvd(cry, full_matrices=False)
            ry[i] = min(ry[i], rmax)
            q = u[:, :ry[i]]
            ind = rect_maxvol(q)[0]  # maxvol(q)
            ry[i] = ind.size
            w1 = mkron(np.ones((ry[i + 1], 1), dtype=np.int), grid[i])
            if np.size(Jy[i + 1]) == 0:
                w2 = np.zeros((n[i] * ry[i + 1], 0), dtype=np.int)
            else:
                w2 = mkron(Jy[i + 1], np.ones((n[i], 1), dtype=np.int))
            Jy[i] = np.hstack((w1, w2))
            Jy[i] = reshape(Jy[i], (n[i] * ry[i + 1], -1))
            Jy[i] = Jy[i][ind, :]
            phi_right[i] = np.tensordot(cores[i], phi_right[i + 1], 1)
            phi_right[i] = reshape(phi_right[i], (-1, n[i] * ry[i + 1]))
            phi_right[i] = phi_right[i][:, ind]

        if dirn > 0 and i < d - 1:
            cry = reshape(cry, (ry[i] * n[i], ry[i + 1]))
            q, r = np.linalg.qr(cry)
            #ind = maxvol(q)
            ind = rect_maxvol(q)[0]
            ry[i + 1] = ind.size
            phi_left[i + 1] = np.tensordot(phi_left[i], cores[i], 1)
            phi_left[i + 1] = reshape(phi_left[i + 1], (ry[i] * n[i], -1))
            phi_left[i + 1] = phi_left[i + 1][ind, :]
            w1 = mkron(np.ones((n[i], 1), dtype=np.int), Jy[i])
            w2 = mkron(grid[i], np.ones((ry[i], 1), dtype=np.int))
            Jy[i + 1] = np.hstack((w1, w2))
            Jy[i + 1] = reshape(Jy[i + 1], (ry[i] * n[i], -1))
            Jy[i + 1] = Jy[i + 1][ind, :]

        i += dirn
        if i == d or i == -1:
            dirn = -dirn
            i += dirn
            swp = swp + 1
    return val, x_full
710441
def get_kl_loss(self, means, log_vars, means_p=None, log_vars_p=None):
    """"""
    kl_loss = 0.0
    if means_p is None:
      means_p = tf.unstack(tf.zeros_like(means))
    if log_vars_p is None:
      log_vars_p = tf.unstack(tf.zeros_like(log_vars))
    enumerated_inputs = enumerate(zip(means, log_vars, means_p, log_vars_p))
    if self.is_training and self.hparams.stochastic_model:
      for i, (mean, log_var, mean_p, log_var_p) in enumerated_inputs:
        kl_loss += common_layers.kl_divergence(mean, log_var, mean_p, log_var_p)
        tf.summary.histogram("posterior_mean_%d" % i, mean)
        tf.summary.histogram("posterior_log_var_%d" % i, log_var)
        tf.summary.histogram("prior_mean_%d" % i, mean_p)
        tf.summary.histogram("prior_log_var_%d" % i, log_var_p)
      tf.summary.scalar("kl_raw", tf.reduce_mean(kl_loss))

    beta = self.get_beta(kl_loss)
    # information capacity from "Understanding disentangling in beta-VAE"
    if self.hparams.information_capacity > 0.0:
      kl_loss = tf.abs(kl_loss - self.hparams.information_capacity)
    return beta * kl_loss
1038126
def metric_tensor(self) -> np.ndarray:
        
        return dot(self._matrix, self._matrix.T)
973105
def log_train(batch_id, batch_num, metric, step_loss, log_interval, epoch_id, learning_rate):
    
    metric_nm, metric_val = metric.get()
    if not isinstance(metric_nm, list):
        metric_nm = [metric_nm]
        metric_val = [metric_val]

    train_str = '[Epoch %d Batch %d/%d] loss=%.4f, lr=%.7f, metrics:' + \
                ','.join([i + ':%.4f' for i in metric_nm])
    logging.info(train_str, epoch_id + 1, batch_id + 1, batch_num, \
                 step_loss / log_interval, \
                 learning_rate, \
                 *metric_val)
621308
def SGD(sym, data_inputs, X, Y, X_test, Y_test, total_iter_num,
        lr=None,
        lr_scheduler=None, prior_precision=1,
        out_grad_f=None,
        initializer=None,
        minibatch_size=100, dev=mx.gpu()):
    """"""
    if out_grad_f is None:
        label_key = list(set(data_inputs.keys()) - set(['data']))[0]
    exe, params, params_grad, _ = get_executor(sym, dev, data_inputs, initializer)
    optimizer = mx.optimizer.create('sgd', learning_rate=lr,
                                    rescale_grad=X.shape[0] / minibatch_size,
                                    lr_scheduler=lr_scheduler,
                                    wd=prior_precision)
    updater = mx.optimizer.get_updater(optimizer)
    start = time.time()
    for i in range(total_iter_num):
        indices = numpy.random.randint(X.shape[0], size=minibatch_size)
        X_batch = X[indices]
        Y_batch = Y[indices]
        exe.arg_dict['data'][:] = X_batch
        if out_grad_f is None:
            exe.arg_dict[label_key][:] = Y_batch
            exe.forward(is_train=True)
            exe.backward()
        else:
            exe.forward(is_train=True)
            exe.backward(out_grad_f(exe.outputs, nd.array(Y_batch, ctx=dev)))
        for k in params:
            updater(k, params_grad[k], params[k])
        if (i + 1) % 500 == 0:
            end = time.time()
            print("Current Iter Num: %d" % (i + 1), "Time Spent: %f" % (end - start))
            sample_test_acc(exe, X=X_test, Y=Y_test, label_num=10, minibatch_size=100)
            start = time.time()
    return exe, params, params_grad
1364603
def log_metrics(self,
                    train_metrics: dict,
                    val_metrics: dict = None,
                    epoch: int = None,
                    log_to_console: bool = False) -> None:
        
        metric_names = set(train_metrics.keys())
        if val_metrics is not None:
            metric_names.update(val_metrics.keys())
        val_metrics = val_metrics or {}

        # For logging to the console
        if log_to_console:
            dual_message_template = "%s |  %8.3f  |  %8.3f"
            no_val_message_template = "%s |  %8.3f  |  %8s"
            no_train_message_template = "%s |  %8s  |  %8.3f"
            header_template = "%s |  %-10s"
            name_length = max([len(x) for x in metric_names])
            logger.info(header_template, "Training".rjust(name_length + 13), "Validation")

        for name in metric_names:
            # Log to tensorboard
            train_metric = train_metrics.get(name)
            if train_metric is not None:
                self.add_train_scalar(name, train_metric, timestep=epoch)
            val_metric = val_metrics.get(name)
            if val_metric is not None:
                self.add_validation_scalar(name, val_metric, timestep=epoch)

            # And maybe log to console
            if log_to_console and val_metric is not None and train_metric is not None:
                logger.info(dual_message_template, name.ljust(name_length), train_metric, val_metric)
            elif log_to_console and val_metric is not None:
                logger.info(no_train_message_template, name.ljust(name_length), "N/A", val_metric)
            elif log_to_console and train_metric is not None:
                logger.info(no_val_message_template, name.ljust(name_length), train_metric, "N/A")
420933
def get_rmse_log(net, X_train, y_train):
    """"""
    num_train = X_train.shape[0]
    clipped_preds = nd.clip(net(X_train), 1, float('inf'))
    return np.sqrt(2 * nd.sum(square_loss(
        nd.log(clipped_preds), nd.log(y_train))).asscalar() / num_train)
1269229
def relax_isometry(self):
        """"""
        for ii in range(self.relaxation_kwds['niter']):
            self.H = self.compute_dual_rmetric()

            self.loss = self.rieman_loss()
            self.trace_var.update(ii,self.H,self.Y,self.eta,self.loss)
            self.trace_var.print_report(ii)
            self.trace_var.save_backup(ii)

            self.compute_gradient()

            self.make_optimization_step(first_iter=(ii == 0))

        self.H = self.compute_dual_rmetric()

        self.trace_var.update(-1,self.H,self.Y,self.eta,self.loss)
        self.trace_var.print_report(ii)
        tracevar_path = os.path.join(self.trace_var.backup_dir, 'results.pyc')
        TracingVariable.save(self.trace_var,tracevar_path)
242325
function (Y) {
			var N = this.N;
			var dim = this.dim; // dim of output space
			var P = this.P;

			var pmul = this.iter < 100 ? 4 : 1; // trick that helps with local optima

			// compute current Q distribution, unnormalized first
			var Qu = zeros(N * N);
			var qsum = 0.0;
			for (var i = 0; i < N; i++) {
				for (var j = i + 1; j < N; j++) {
					var dsum = 0.0;
					for (var d = 0; d < dim; d++) {
						var dhere = Y[i][d] - Y[j][d];
						dsum += dhere * dhere;
					}
					var qu = 1.0 / (1.0 + dsum); // Student t-distribution
					Qu[i * N + j] = qu;
					Qu[j * N + i] = qu;
					qsum += 2 * qu;
				}
			}
			// normalize Q distribution to sum to 1
			var NN = N * N;
			var Q = zeros(NN);
			for (var q = 0; q < NN; q++) {
				Q[q] = Math.max(Qu[q] / qsum, 1e-100);
			}

			var cost = 0.0;
			var grad = [];
			for (var i = 0; i < N; i++) {
				var gsum = new Array(dim); // init grad for point i
				for (var d = 0; d < dim; d++) {
					gsum[d] = 0.0;
				}
				for (var j = 0; j < N; j++) {
					cost += -P[i * N + j] * Math.log(Q[i * N + j]); // accumulate cost (the non-constant portion at least...)
					var premult = 4 * (pmul * P[i * N + j] - Q[i * N + j]) * Qu[i * N + j];
					for (var d = 0; d < dim; d++) {
						gsum[d] += premult * (Y[i][d] - Y[j][d]);
					}
				}
				grad.push(gsum);
			}

			return {
				cost: cost,
				grad: grad
			};
		}
552411
def examples(self):
        
        # TODO: adapt for tensor-valued functions

        # Get the points and calculate some statistics on them
        mins = self.domain.min()
        maxs = self.domain.max()
        means = (maxs + mins) / 2.0
        stds = (maxs - mins) / 4.0
        ndim = getattr(self.domain, 'ndim', 0)

        # Zero and One
        yield ('Zero', self.zero())
        yield ('One', self.one())

        # Indicator function in first dimension
        def step_fun(x):
            return (x[0] > means[0])

        yield ('Step', self.element(step_fun))

        # Indicator function on hypercube
        def cube_fun(x):
            result = True
            for points, mean, std in zip(x, means, stds):
                result = np.logical_and(result, points < mean + std)
                result = np.logical_and(result, points > mean - std)
            return result

        yield ('Cube', self.element(cube_fun))

        # Indicator function on a ball
        if ndim > 1:  # Only if ndim > 1, don't duplicate cube
            def ball_fun(x):
                r = sum((xi - mean) ** 2 / std ** 2
                        for xi, mean, std in zip(x, means, stds))
                return r < 1.0

            yield ('Ball', self.element(ball_fun))

        # Gaussian function
        def gaussian_fun(x):
            r2 = sum((xi - mean) ** 2 / (2 * std ** 2)
                     for xi, mean, std in zip(x, means, stds))
            return np.exp(-r2)

        yield ('Gaussian', self.element(gaussian_fun))

        # Gradient in each dimensions
        for axis in range(ndim):
            def gradient_fun(x):
                return (x[axis] - mins[axis]) / (maxs[axis] - mins[axis])

            yield ('Grad {}'.format(axis), self.element(gradient_fun))

        # Gradient in all dimensions
        if ndim > 1:  # Only if ndim > 1, don't duplicate grad 0
            def all_gradient_fun(x):
                return sum((xi - xmin) / (xmax - xmin)
                           for xi, xmin, xmax in zip(x, mins, maxs))

            yield ('Grad all', self.element(all_gradient_fun))
237852
def add_log_hooks_to_pytorch_module(self, module, name=None, prefix='', log_parameters=True, log_gradients=True, log_freq=0):
        
        if name is not None:
            prefix = prefix + name

        if log_parameters:
            def parameter_log_hook(module, input_, output, log_track):
                if not log_track_update(log_track):
                    return
                for name, parameter in module.named_parameters():
                    # for pytorch 0.3 Variables
                    if isinstance(parameter, torch.autograd.Variable):
                        data = parameter.data
                    else:
                        data = parameter
                    self.log_tensor_stats(
                        data.cpu(), 'parameters/' + prefix + name)
            log_track_params = log_track_init(log_freq)
            module.register_forward_hook(
                lambda mod, inp, outp: parameter_log_hook(mod, inp, outp, log_track_params))

        if log_gradients:
            for name, parameter in module.named_parameters():
                if parameter.requires_grad:
                    log_track_grad = log_track_init(log_freq)
                    self._hook_variable_gradient_stats(
                        parameter, 'gradients/' + prefix + name, log_track_grad)
297607
def lognormcdf(x, mu, tau):
    """"""
    x = np.atleast_1d(x)
    return np.array(
        [0.5 * (1 - flib.derf(-(np.sqrt(tau / 2)) * (np.log(y) - mu))) for y in x])