Score based on the loss function
@param model the model to score with
@param testData the test data to score
@param average whether to average the score
for the whole batch or not
@return the score for the given test set
199794
public static double score(ComputationGraph model, MultiDataSetIterator testData, boolean average) {
        //TODO: do this properly taking into account division by N, L1/L2 etc
        double sumScore = 0.0;
        int totalExamples = 0;
        while (testData.hasNext()) {
            MultiDataSet ds = testData.next();
            long numExamples = ds.getFeatures(0).size(0);
            sumScore += numExamples * model.score(ds);
            totalExamples += numExamples;
        }

        if (!average)
            return sumScore;
        return sumScore / totalExamples;
    }
750554
def get_scores_and_p_values(self, tdm, category):
		'''
		
		'''
		X = tdm._X
		y = self._make_response_variable_1_or_negative_1(category, tdm)
		pX = X / X.sum(axis=1)
		ansX = self._anscombe_transform(pX.copy())
		B, istop, itn, r1norm, r2norm, anorm, acond, arnorm, xnorm, var\
			= lsqr(A=ansX, b=y, calc_var=True)
183426
@Override
    public double f1Score(INDArray input, INDArray labels) {
        feedForward(input);
        setLabels(labels);
        Evaluation eval = new Evaluation();
        eval.eval(labels, output(input));
        return eval.f1();
    }
683469
public <T extends RegressionEvaluation> T evaluateRegression(JavaRDD<DataSet> data) {
        return evaluateRegression(data, DEFAULT_EVAL_SCORE_BATCH_SIZE);
    }
158264
protected int scoreServiceDocument(ODataRequestContext requestContext, MediaType requiredMediaType) {
        if (isServiceDocument(requestContext.getUri())) {
            int scoreByFormat = scoreByFormat(getFormatOption(requestContext.getUri()), requiredMediaType);
            int scoreByMediaType = scoreByMediaType(requestContext.getRequest().getAccept(), requiredMediaType);
            return max(scoreByFormat, scoreByMediaType);
        } else {
            return DEFAULT_SCORE;
        }
    }
658515
public static double score(MultiLayerNetwork model, DataSetIterator testSet, RegressionValue regressionValue) {
        RegressionEvaluation eval = model.evaluateRegression(testSet);
        return getScoreFromRegressionEval(eval, regressionValue);
    }
88702
protected static double computeScore(LinkedList<double[]> scoreList, int tag)
    {
        double score = 0;
        for (double[] w : scoreList)
        {
            score += w[tag];
        }
        return score;
    }
674846
def train_model(model, train_data_loader, val_data_loader, embedding, ctx, args):
    
    logger.info(vars(args))

    # Initialization
    model.hybridize()
    model.collect_params().initialize(mx.init.Normal(0.01), ctx=ctx)
    model.word_emb.weight.set_data(embedding.idx_to_vec)
    # Fix word embedding
    model.word_emb.weight.grad_req = 'null'

    loss_func = gluon.loss.SoftmaxCrossEntropyLoss()
    trainer = gluon.Trainer(model.collect_params(), 'adagrad',
                            {'learning_rate': args.lr,
                             'wd': args.weight_decay,
                             'clip_gradient': 5})

    checkpoints_dir = os.path.join(args.output_dir, 'checkpoints')
    if not os.path.exists(checkpoints_dir):
        os.makedirs(checkpoints_dir)

    best_val_acc = 0.
    for epoch_id in range(args.epochs):
        avg_loss = 0.
        avg_acc = 0.
        for batch_id, example in enumerate(train_data_loader):
            s1, s2, label = example
            s1 = s1.as_in_context(ctx)
            s2 = s2.as_in_context(ctx)
            label = label.as_in_context(ctx)

            with autograd.record():
                output = model(s1, s2)
                loss = loss_func(output, label).mean()
            loss.backward()
            trainer.step(1)
            avg_loss += loss.sum().asscalar()

            pred = output.argmax(axis=1)
            acc = (pred == label.astype(np.float32)).mean()
            avg_acc += acc.asscalar()

            if (batch_id + 1) % args.print_interval == 0:
                avg_loss /= args.print_interval
                avg_acc /= args.print_interval
                logger.info('[Epoch {} Batch {}/{}] loss={:.4f}, acc={:.4f}'
                            .format(epoch_id, batch_id + 1, len(train_data_loader),
                                    avg_loss, avg_acc))
                avg_loss = 0.
                avg_acc = 0.

        # Validation
        val_loss, val_acc = test_model(model, val_data_loader, loss_func, ctx)
        if val_acc > best_val_acc:
            best_val_acc = val_acc
            checkpoint_path = os.path.join(args.output_dir, 'checkpoints', 'valid_best.params')
            model.save_parameters(checkpoint_path)
        logger.info('[Epoch {}] valid loss={:.4f}, valid acc={:.4f}, best valid acc={:.4f}'
                    .format(epoch_id, val_loss, val_acc, best_val_acc))

        # Save checkpoint of last epoch
        checkpoint_path = os.path.join(args.output_dir, 'checkpoints', 'last.params')
        model.save_parameters(checkpoint_path)
291118
public RandomVariable getRate(AnalyticModel model) {
		if(model==null) {
			throw new IllegalArgumentException("model==null");
		}

		DiscountCurveInterface discountCurve = model.getDiscountCurve(discountCurveName);
		if(discountCurve == null) {
			throw new IllegalArgumentException("No discount curve with name '" + discountCurveName + "' was found in the model:\n" + model.toString());
		}

		double payoutDate = schedule.getPeriodStart(0);
		double maturity = schedule.getPayment(0);
		double periodLength = schedule.getPeriodLength(0);

		RandomVariable discountFactor = discountCurve.getDiscountFactor(model, maturity);
		RandomVariable discountFactorPayout = discountCurve.getDiscountFactor(model, payoutDate);

		return discountFactorPayout.div(discountFactor).sub(1).div(periodLength);
	}
1373201
def __score_method(X, y, fcounts, model_generator, score_function, method_name, nreps=10, test_size=100, cache_dir="/tmp"):
    

    old_seed = np.random.seed()
    np.random.seed(3293)

    # average the method scores over several train/test splits
    method_reps = []

    data_hash = hashlib.sha256(__toarray(X).flatten()).hexdigest() + hashlib.sha256(__toarray(y)).hexdigest()
    for i in range(nreps):
        X_train, X_test, y_train, y_test = train_test_split(__toarray(X), y, test_size=test_size, random_state=i)

        # define the model we are going to explain, caching so we onlu build it once
        model_id = "model_cache__v" + "__".join([__version__, data_hash, model_generator.__name__])+".pickle"
        cache_file = os.path.join(cache_dir, model_id + ".pickle")
        if os.path.isfile(cache_file):
            with open(cache_file, "rb") as f:
                model = pickle.load(f)
        else:
            model = model_generator()
            model.fit(X_train, y_train)
            with open(cache_file, "wb") as f:
                pickle.dump(model, f)

        attr_key = "_".join([model_generator.__name__, method_name, str(test_size), str(nreps), str(i), data_hash])
        def score(attr_function):
            def cached_attr_function(X_inner):
                if attr_key not in _attribution_cache:
                    _attribution_cache[attr_key] = attr_function(X_inner)
                return _attribution_cache[attr_key]

            #cached_attr_function = lambda X: __check_cache(attr_function, X)
            if fcounts is None:
                return score_function(X_train, X_test, y_train, y_test, cached_attr_function, model, i)
            else:
                scores = []
                for f in fcounts:
                    scores.append(score_function(f, X_train, X_test, y_train, y_test, cached_attr_function, model, i))
                return np.array(scores)

        # evaluate the method (only building the attribution function if we need to)
        if attr_key not in _attribution_cache:
            method_reps.append(score(getattr(methods, method_name)(model, X_train)))
        else:
            method_reps.append(score(None))

    np.random.seed(old_seed)
    return np.array(method_reps).mean(0)
957102
public void setSampling( double min , double max , int total ) {
		this.sampleMin = min;
		this.sampleMax = max;
		this.numSamples = total;
		this.scores = new double[numSamples];
	}
471231
@Override
  public final double[] score0(double[] dataRow, double offset, double[] preds) {
    assert(dataRow != null) : "doubles are null"; // check to make sure data is not null
    double[] neuronsInput = new double[_units[0]]; // store inputs into the neural network
    double[] neuronsOutput;  // save output from a neural network layer

    // transform inputs: NAs in categoricals are always set to new extra level.
    setInput(dataRow, neuronsInput, _numsA, _catsA, _nums, _cats, _catoffsets, _normmul, _normsub, _use_all_factor_levels, true);

    // proprogate inputs through neural network
    for (int layer=0; layer < _numLayers; layer++) {
      NeuralNetwork oneLayer = new NeuralNetwork(_allActivations[layer], _all_drop_out_ratios[layer],
              _weightsAndBias[layer], neuronsInput, _units[layer + 1]);
      neuronsOutput = oneLayer.fprop1Layer();
      neuronsInput = neuronsOutput;
    }
    if (!this.isAutoEncoder())
      assert (_nclasses == neuronsInput.length) : "nclasses " + _nclasses + " neuronsOutput.length " + neuronsInput.length;
    // Correction for classification or standardize outputs
    return modifyOutputs(neuronsInput, preds, dataRow);
  }
73186
@SuppressWarnings({"ForLoopReplaceableByForEach"})
	protected void selectMatchSet(List<Point> dataSet, double threshold, Model param) {
		candidatePoints.clear();
		modelDistance.setModel(param);

		for (int i = 0; i < dataSet.size(); i++) {
			Point point = dataSet.get(i);

			double distance = modelDistance.computeDistance(point);
			if (distance < threshold) {
				matchToInput[candidatePoints.size()] = i;
				candidatePoints.add(point);
			}
		}
	}
1334667
def update_average_model(self, model):
        
        for model_param, average_param in zip(model.parameters(), self.average_model.parameters()):
            # EWMA average model update
            average_param.data.mul_(self.average_model_alpha).add_(model_param.data * (1 - self.average_model_alpha))
1233404
@Override protected void queryArgumentValueSet(Argument arg, java.util.Properties inputArgs) {
    super.queryArgumentValueSet(arg, inputArgs);

    if (!arg._name.equals("checkpoint") && !Utils.contains(cp_modifiable, arg._name)) {
      if (checkpoint != null) {
        arg.disable("Taken from model checkpoint.");
        final DeepLearningModel cp_model = UKV.get(checkpoint);
        if (cp_model == null) {
          throw new IllegalArgumentException("Checkpointed model was not found.");
        }
        if (cp_model.model_info().unstable()) {
          throw new IllegalArgumentException("Checkpointed model was unstable. Not restarting.");
        }
        return;
      }
    }
    if(arg._name.equals("initial_weight_scale") &&
            (initial_weight_distribution == InitialWeightDistribution.UniformAdaptive)
            ) {
      arg.disable("Using sqrt(6 / (# units + # units of previous layer)) for Uniform distribution.", inputArgs);
    }
    if (classification) {
      if(arg._name.equals("regression_stop")) {
        arg.disable("Only for regression.", inputArgs);
      }
      if((arg._name.equals("max_after_balance_size") || arg._name.equals("class_sampling_factors")) && !balance_classes) {
        arg.disable("Requires balance_classes.", inputArgs);
      }
    }
    else {
      if(arg._name.equals("classification_stop")
              || arg._name.equals("max_confusion_matrix_size")
              || arg._name.equals("max_hit_ratio_k")
              || arg._name.equals("max_after_balance_size")
              || arg._name.equals("balance_classes")
              || arg._name.equals("class_sampling_factors")
              ) {
        arg.disable("Only for classification.", inputArgs);
      }
      if (validation != null && arg._name.equals("score_validation_sampling")) {
        score_validation_sampling = ClassSamplingMethod.Uniform;
        arg.disable("Using uniform sampling for validation scoring dataset.", inputArgs);
      }
    }
    if ((arg._name.equals("score_validation_samples") || arg._name.equals("score_validation_sampling")) && validation == null) {
      arg.disable("Requires a validation data set.", inputArgs);
    }
    if (Utils.contains(expert_options, arg._name) && !expert_mode) {
      arg.disable("Only in expert mode.", inputArgs);
    }
    if (!adaptive_rate) {
      if (arg._name.equals("rho") || arg._name.equals("epsilon")) {
        arg.disable("Only for adaptive learning rate.", inputArgs);
        rho = 0;
        epsilon = 0;
      }
    } else {
      if (arg._name.equals("rate") || arg._name.equals("rate_annealing") || arg._name.equals("rate_decay") || arg._name.equals("nesterov_accelerated_gradient")
              || arg._name.equals("momentum_start") || arg._name.equals("momentum_ramp") || arg._name.equals("momentum_stable") ) {
        arg.disable("Only for non-adaptive learning rate.", inputArgs);
        momentum_start = 0;
        momentum_stable = 0;
      }
    }
    if (arg._name.equals("hidden_dropout_ratios")) {
      if (activation != Activation.TanhWithDropout && activation != Activation.MaxoutWithDropout && activation != Activation.RectifierWithDropout) {
        arg.disable("Only for activation functions with dropout.", inputArgs);
      }
    }
    if (arg._name.equals("replicate_training_data") && (H2O.CLOUD.size() == 1)) {
      arg.disable("Only for multi-node operation.");
      replicate_training_data = false;
    }
    if (arg._name.equals("single_node_mode") && (H2O.CLOUD.size() == 1 || !replicate_training_data)) {
      arg.disable("Only for multi-node operation with replication.");
      single_node_mode = false;
    }
    if (arg._name.equals("use_all_factor_levels") && autoencoder ) {
      arg.disable("Automatically enabled for auto-encoders.");
      use_all_factor_levels = true;
    }
    if(arg._name.equals("override_with_best_model") && n_folds != 0) {
      arg.disable("Only without n-fold cross-validation.", inputArgs);
      override_with_best_model = false;
    }
    if(arg._name.equals("average_activation") && !autoencoder) {
      arg.disable("Only for autoencoder.", inputArgs);
    }
    if(arg._name.equals("sparsity_beta") && !autoencoder) {
      arg.disable("Only for autoencoder.", inputArgs);
    }
  }
694946
public static void getAttributeScores(String modelPath, String lexiconF,
			int topAttributesNumber) throws IOException {
		// load the model + the lexicon
		// try to see if we can get a list of the best scores from the model
		// works only for liblinear
		Lexicon lexicon = new Lexicon(lexiconF);
		Model liblinearModel = Model.load(new File(modelPath));
		double[] weights = liblinearModel.getFeatureWeights();
		// dump all the weights
		int numClasses = liblinearModel.getNrClass();
		int numFeatures = liblinearModel.getNrFeature();

		Map<Integer, String> invertedAttributeIndex = lexicon
				.getInvertedIndex();

		Map<String, WeightedAttributeQueue> topAttributesPerLabel = new HashMap<String, WeightedAttributeQueue>(
				numClasses);

		// for (int i = 0; i < nr_w; i++) {
		// double contrib = w[(idx - 1) * nr_w + i] * lx.getValue();
		// }
		//
		// idx 1 in class 1 -> 0 x 22 + 0 = 0
		// idx 2 in class 1 -> 1 x 22 + 0 = 22
		// idx 1 in class 2 -> 0 x 22 + 1 = 1
		// idx 2 in class 2 -> 1 x 22 + 1 = 23

		// initialise the queues
		if (topAttributesNumber != -1) {
			for (int classNum = 0; classNum < numClasses; classNum++) {
				String classLabel = lexicon.getLabel(classNum);
				WeightedAttributeQueue queue = new WeightedAttributeQueue(
						topAttributesNumber);
				topAttributesPerLabel.put(classLabel, queue);
			}
		}

		for (int classNum = 0; classNum < numClasses; classNum++) {
			String classLabel = lexicon.getLabel(classNum);
			WeightedAttributeQueue queue = topAttributesPerLabel
					.get(classLabel);
			for (int featNum = 0; featNum < numFeatures; featNum++) {
				int pos = featNum * numClasses + classNum;
				double featWeight = weights[pos];
				String attLabel = invertedAttributeIndex.get(featNum + 1);

				// display the values between -0.001 and +0.001 as 0
				if (featWeight < 0.001 && featWeight > -0.001)
					featWeight = 0;
				// want to limit to the top n terms?
				if (topAttributesNumber != -1) {
					WeightedAttribute wa = new WeightedAttribute(attLabel,
							featWeight);
					queue.insertWithOverflow(wa);
					continue;
				}

				System.out.println(attLabel + "\t" + classLabel + "\t"
						+ featWeight);
			}
		}

		// dump the attributes per label
		if (topAttributesNumber < 1)
			return;

		Iterator<String> labelIter = topAttributesPerLabel.keySet().iterator();
		while (labelIter.hasNext()) {
			String label = (String) labelIter.next();
			System.out.println("LABEL : " + label);
			WeightedAttributeQueue queue = topAttributesPerLabel.get(label);
			// revert the order
			String[] sorted = new String[queue.size()];

			for (int i = queue.size() - 1; i >= 0; i--) {
				WeightedAttribute wa = queue.pop();
				sorted[i] = wa.label + " : " + wa.weight;
			}

			for (int j = 0; j < sorted.length; j++) {
				System.out.println(j + 1 + "\t" + sorted[j]);
			}
		}
	}
440719
@SuppressWarnings("unused") // called through reflection by RequestServer
  public ModelMetricsListSchemaV3 score(int version, ModelMetricsListSchemaV3 s) {
    // parameters checking:
    if (null == s.model) throw new H2OIllegalArgumentException("model", "predict", s.model);
    if (null == DKV.get(s.model.name)) throw new H2OKeyNotFoundArgumentException("model", "predict", s.model.name);

    if (null == s.frame) throw new H2OIllegalArgumentException("frame", "predict", s.frame);
    if (null == DKV.get(s.frame.name)) throw new H2OKeyNotFoundArgumentException("frame", "predict", s.frame.name);

    ModelMetricsList parms = s.createAndFillImpl();

    String customMetricFunc = s.custom_metric_func;
    if (customMetricFunc == null) {
      customMetricFunc = parms._model._parms._custom_metric_func;
    }
    parms._model.score(parms._frame, parms._predictions_name, null, true, CFuncRef.from(customMetricFunc)).remove(); // throw away predictions, keep metrics as a side-effect
    ModelMetricsListSchemaV3 mm = this.fetch(version, s);

    // TODO: for now only binary predictors write an MM object.
    // For the others cons one up here to return the predictions frame.
    if (null == mm)
      mm = new ModelMetricsListSchemaV3();

    if (null == mm.model_metrics || 0 == mm.model_metrics.length) {
      Log.warn("Score() did not return a ModelMetrics for model: " + s.model + " on frame: " + s.frame);
    }

    return mm;
  }