Evaluate the network, with optional metadata

@param labels   Data labels
@param predictions        Network predictions
@param recordMetaData Optional; may be null. If not null, should have size equal to the number of outcomes/guesses
428521
@Override
    public void eval(INDArray labels, INDArray predictions, INDArray mask, final List<? extends Serializable> recordMetaData) {

        Triple<INDArray,INDArray, INDArray> p = BaseEvaluation.reshapeAndExtractNotMasked(labels, predictions, mask, axis);
        if(p == null){
            //All values masked out; no-op
            return;
        }

        INDArray labels2d = p.getFirst();
        INDArray predictions2d = p.getSecond();
        INDArray maskArray = p.getThird();
        Preconditions.checkState(maskArray == null, "Per-output masking for ROCMultiClass is not supported");


        if(labels2d.dataType() != predictions2d.dataType())
            labels2d = labels2d.castTo(predictions2d.dataType());

        // FIXME: int cast
        int n = (int) labels2d.size(1);
        if (underlying == null) {
            underlying = new ROC[n];
            for (int i = 0; i < n; i++) {
                underlying[i] = new ROC(thresholdSteps, rocRemoveRedundantPts);
            }
        }

        if (underlying.length != labels2d.size(1)) {
            throw new IllegalArgumentException(
                            "Cannot evaluate data: number of label classes does not match previous call. " + "Got "
                                            + labels2d.size(1) + " labels (from array shape "
                                            + Arrays.toString(labels2d.shape()) + ")"
                                            + " vs. expected number of label classes = " + underlying.length);
        }

        for (int i = 0; i < n; i++) {
            INDArray prob = predictions2d.getColumn(i, true); //Probability of class i
            INDArray label = labels2d.getColumn(i, true);
            //Workaround for: https://github.com/deeplearning4j/deeplearning4j/issues/7305
            if(prob.rank() == 0)
                prob = prob.reshape(1,1);
            if(label.rank() == 0)
                label = label.reshape(1,1);
            underlying[i].eval(label, prob);
        }
    }
220008
public void evaluate(final ObjectStream<SequenceLabelSample> samples,
      final int nFolds) throws IOException {

    // Note: The name samples need to be grouped on a document basis.

    final CrossValidationPartitioner<DocumentSample> partitioner = new CrossValidationPartitioner<DocumentSample>(
        new NameToDocumentSampleStream(samples), nFolds);

    while (partitioner.hasNext()) {

      final CrossValidationPartitioner.TrainingSampleStream<DocumentSample> trainingSampleStream = partitioner
          .next();

      SequenceLabelerModel model = null;
      if (this.factory != null) {
        model = SequenceLabelerME.train(this.languageCode,
            new DocumentToNameSampleStream(trainingSampleStream), this.params,
            this.factory);
      } else {
        System.err.println("You need to implement a SequenceLabelerFactory!");
        System.exit(1);
      }
      
      String corpusFormat = Flags.getCorpusFormat(this.params);
      // do testing
      final SequenceLabelerEvaluator evaluator = new SequenceLabelerEvaluator(corpusFormat,
          new SequenceLabelerME(model), this.listeners);

      evaluator.evaluate(new DocumentToNameSampleStream(
          trainingSampleStream.getTestSampleStream()));

      this.fmeasure.mergeInto(evaluator.getFMeasure());
    }
  }
372657
public void setColumnMetadata(java.util.Collection<ColumnMetadata> columnMetadata) {
        if (columnMetadata == null) {
            this.columnMetadata = null;
            return;
        }

        this.columnMetadata = new java.util.ArrayList<ColumnMetadata>(columnMetadata);
    }
1331422
@Override
    public void filterAndStrip(int[] labels) {
        DataSet filtered = filterBy(labels);
        List<Integer> newLabels = new ArrayList<>();

        //map new labels to index according to passed in labels
        Map<Integer, Integer> labelMap = new HashMap<>();

        for (int i = 0; i < labels.length; i++)
            labelMap.put(labels[i], i);

        //map examples
        for (int i = 0; i < filtered.numExamples(); i++) {
            DataSet example = filtered.get(i);
            int o2 = example.outcome();
            Integer outcome = labelMap.get(o2);
            newLabels.add(outcome);

        }


        INDArray newLabelMatrix = Nd4j.create(filtered.numExamples(), labels.length);

        if (newLabelMatrix.rows() != newLabels.size())
            throw new IllegalStateException("Inconsistent label sizes");

        for (int i = 0; i < newLabelMatrix.rows(); i++) {
            Integer i2 = newLabels.get(i);
            if (i2 == null)
                throw new IllegalStateException("Label not found on row " + i);
            INDArray newRow = FeatureUtil.toOutcomeVector(i2, labels.length);
            newLabelMatrix.putRow(i, newRow);

        }

        setFeatures(filtered.getFeatures());
        setLabels(newLabelMatrix);
    }
544555
public void setLabelNames(List<String> labels) {
        if (labels == null) {
            this.labels = null;
            return;
        }
        this.labels = new ArrayList<>(labels);
    }
891135
public void storeLabel(String label) {
        if (labels == null)
            labels = new ArrayList<>();

        if (!uniq.contains(label)) {
            uniq.add(label);
            labels.add(label);
        }
    }
1253281
public void setLabel(String label) {

        int ix = lstSizes.indexOf(label);
        if (ix != -1) {
            setLabel(ix);
        }
    }
1188200
public void setRecords(java.util.Collection<Record> records) {
        if (records == null) {
            this.records = null;
            return;
        }

        this.records = new com.amazonaws.internal.SdkInternalList<Record>(records);
    }
907151
public void setLabels(String[] outcomeLabels)
    {
        this.numOutcomes = outcomeLabels.length;
        r = Math.log(1.0 / numOutcomes);
    }
474329
public void setLabel(String label) {
		final boolean entryExitLogging = log.isLoggable(Level.FINER);
		final String methodName = "setLabel"; //$NON-NLS-1$
		if (entryExitLogging) {
			log.entering(DependencyList.class.getName(), methodName, new Object[]{label});
		}
		this.label = label;
		if (entryExitLogging) {
			log.exiting(DependencyList.class.getName(), methodName);
		}
	}
242272
public static Map<String, String> getOrCreateLabels(HasMetadata entity) {
    ObjectMeta metadata = getOrCreateMetadata(entity);
    Map<String, String> answer = metadata.getLabels();
    if (answer == null) {
      // use linked so the annotations can be in the FIFO order
      answer = new LinkedHashMap<>();
      metadata.setLabels(answer);
    }
    return answer;
  }
1162099
public function score(array $predictions, array $labels) : float
    {
        if (empty($predictions)) {
            return 0.;
        }

        if (count($predictions) !== count($labels)) {
            throw new InvalidArgumentException('The number of labels'
                . ' must equal the number of predictions.');
        }

        $classes = array_unique(array_merge($predictions, $labels));

        $truePos = $trueNeg = $falsePos = $falseNeg = array_fill_keys($classes, 0);

        foreach ($predictions as $i => $prediction) {
            $label = $labels[$i];

            if ($prediction === $label) {
                $truePos[$prediction]++;

                foreach ($classes as $class) {
                    if ($class !== $prediction) {
                        $trueNeg[$class]++;
                    }
                }
            } else {
                $falsePos[$prediction]++;
                $falseNeg[$label]++;
            }
        }

        return Stats::mean(
            array_map([$this, 'mcc'], $truePos, $trueNeg, $falsePos, $falseNeg)
        );
    }
662053
@SuppressWarnings("unused") // called through reflection by RequestServer
  public ModelMetricsListSchemaV3 predict(int version, ModelMetricsListSchemaV3 s) {
    // parameters checking:
    if (s.model == null) throw new H2OIllegalArgumentException("model", "predict", null);
    if (DKV.get(s.model.name) == null) throw new H2OKeyNotFoundArgumentException("model", "predict", s.model.name);

    // Aggregator doesn't need a Frame to 'predict'
    if (s.exemplar_index < 0) {
      if (s.frame == null) throw new H2OIllegalArgumentException("frame", "predict", null);
      if (DKV.get(s.frame.name) == null) throw new H2OKeyNotFoundArgumentException("frame", "predict", s.frame.name);
    }

    ModelMetricsList parms = s.createAndFillImpl();

    Frame predictions;
    Frame deviances = null;
    if (!s.reconstruction_error && !s.reconstruction_error_per_feature && s.deep_features_hidden_layer < 0 &&
        !s.project_archetypes && !s.reconstruct_train && !s.leaf_node_assignment && !s.predict_staged_proba && !s.predict_contributions && s.exemplar_index < 0) {
      if (null == parms._predictions_name)
        parms._predictions_name = "predictions" + Key.make().toString().substring(0,5) + "_" + parms._model._key.toString() + "_on_" + parms._frame._key.toString();
      String customMetricFunc = s.custom_metric_func;
      if (customMetricFunc == null) {
        customMetricFunc = parms._model._parms._custom_metric_func;
      }
      predictions = parms._model.score(parms._frame, parms._predictions_name, null, true, CFuncRef.from(customMetricFunc));
      if (s.deviances) {
        if (!parms._model.isSupervised())
          throw new H2OIllegalArgumentException("Deviances can only be computed for supervised models.");
        if (null == parms._deviances_name)
          parms._deviances_name = "deviances" + Key.make().toString().substring(0, 5) + "_" + parms._model._key.toString() + "_on_" + parms._frame._key.toString();
        deviances = parms._model.computeDeviances(parms._frame, predictions, parms._deviances_name);
      }
    } else {
      if (s.deviances)
        throw new H2OIllegalArgumentException("Cannot compute deviances in combination with other special predictions.");
      if (Model.DeepFeatures.class.isAssignableFrom(parms._model.getClass())) {
        if (s.reconstruction_error || s.reconstruction_error_per_feature) {
          if (s.deep_features_hidden_layer >= 0)
            throw new H2OIllegalArgumentException("Can only compute either reconstruction error OR deep features.", "");
          if (null == parms._predictions_name)
            parms._predictions_name = "reconstruction_error" + Key.make().toString().substring(0,5) + "_" + parms._model._key.toString() + "_on_" + parms._frame._key.toString();
          predictions = ((Model.DeepFeatures) parms._model).scoreAutoEncoder(parms._frame, Key.make(parms._predictions_name), parms._reconstruction_error_per_feature);
        } else {
          if (s.deep_features_hidden_layer < 0)
            throw new H2OIllegalArgumentException("Deep features hidden layer index must be >= 0.", "");
          if (null == parms._predictions_name)
            parms._predictions_name = "deep_features" + Key.make().toString().substring(0,5) + "_" + parms._model._key.toString() + "_on_" + parms._frame._key.toString();
          predictions = ((Model.DeepFeatures) parms._model).scoreDeepFeatures(parms._frame, s.deep_features_hidden_layer);
        }
        predictions = new Frame(Key.<Frame>make(parms._predictions_name), predictions.names(), predictions.vecs());
        DKV.put(predictions._key, predictions);
      } else if(Model.GLRMArchetypes.class.isAssignableFrom(parms._model.getClass())) {
        if(s.project_archetypes) {
          if (parms._predictions_name == null)
            parms._predictions_name = "reconstructed_archetypes_" + Key.make().toString().substring(0, 5) + "_" + parms._model._key.toString() + "_of_" + parms._frame._key.toString();
          predictions = ((Model.GLRMArchetypes) parms._model).scoreArchetypes(parms._frame, Key.<Frame>make(parms._predictions_name), s.reverse_transform);
        } else {
          assert s.reconstruct_train;
          if (parms._predictions_name == null)
            parms._predictions_name = "reconstruction_" + Key.make().toString().substring(0, 5) + "_" + parms._model._key.toString() + "_of_" + parms._frame._key.toString();
          predictions = ((Model.GLRMArchetypes) parms._model).scoreReconstruction(parms._frame, Key.<Frame>make(parms._predictions_name), s.reverse_transform);
        }
      } else if(s.leaf_node_assignment) {
        assert(Model.LeafNodeAssignment.class.isAssignableFrom(parms._model.getClass()));
        if (null == parms._predictions_name)
          parms._predictions_name = "leaf_node_assignment" + Key.make().toString().substring(0, 5) + "_" + parms._model._key.toString() + "_on_" + parms._frame._key.toString();
        Model.LeafNodeAssignment.LeafNodeAssignmentType type = null == s.leaf_node_assignment_type ? Model.LeafNodeAssignment.LeafNodeAssignmentType.Path : s.leaf_node_assignment_type;
        predictions = ((Model.LeafNodeAssignment) parms._model).scoreLeafNodeAssignment(parms._frame, type, Key.<Frame>make(parms._predictions_name));
      } else if(s.predict_staged_proba) {
        if (! (parms._model instanceof Model.StagedPredictions)) {
          throw new H2OIllegalArgumentException("Model type " + parms._model._parms.algoName() + " doesn't support Staged Predictions.");
        }
        if (null == parms._predictions_name)
          parms._predictions_name = "staged_proba_" + Key.make().toString().substring(0, 5) + "_" + parms._model._key.toString() + "_on_" + parms._frame._key.toString();
        predictions = ((Model.StagedPredictions) parms._model).scoreStagedPredictions(parms._frame, Key.<Frame>make(parms._predictions_name));
      } else if(s.predict_contributions) {
        if (! (parms._model instanceof Model.Contributions)) {
          throw new H2OIllegalArgumentException("Model type " + parms._model._parms.algoName() + " doesn't support calculating Feature Contributions.");
        }
        if (null == parms._predictions_name)
          parms._predictions_name = "contributions_" + Key.make().toString().substring(0, 5) + "_" + parms._model._key.toString() + "_on_" + parms._frame._key.toString();
        predictions = ((Model.Contributions) parms._model).scoreContributions(parms._frame, Key.<Frame>make(parms._predictions_name));
      } else if(s.exemplar_index >= 0) {
        assert(Model.ExemplarMembers.class.isAssignableFrom(parms._model.getClass()));
        if (null == parms._predictions_name)
          parms._predictions_name = "members_" + parms._model._key.toString() + "_for_exemplar_" + parms._exemplar_index;
        predictions = ((Model.ExemplarMembers) parms._model).scoreExemplarMembers(Key.<Frame>make(parms._predictions_name), parms._exemplar_index);
      }
      else throw new H2OIllegalArgumentException("Requires a Deep Learning, GLRM, DRF or GBM model.", "Model must implement specific methods.");
    }

    ModelMetricsListSchemaV3 mm = this.fetch(version, s);

    // TODO: for now only binary predictors write an MM object.
    // For the others cons one up here to return the predictions frame.
    if (null == mm)
      mm = new ModelMetricsListSchemaV3();

    mm.predictions_frame = new KeyV3.FrameKeyV3(predictions._key);
    if (parms._leaf_node_assignment) //don't show metrics in leaf node assignments are made
      mm.model_metrics = null;
    if (deviances !=null)
      mm.deviances_frame = new KeyV3.FrameKeyV3(deviances._key);

    if (null == mm.model_metrics || 0 == mm.model_metrics.length) {
      // There was no response in the test set -> cannot make a model_metrics object
    } else {
      mm.model_metrics[0].predictions = new FrameV3(predictions, 0, 100); // TODO: Should call schema(version)
    }
    return mm;
  }
736566
@Override
    public boolean validateLabel(String label) {
        Objects.requireNonNull(label, "label cannot be null");
        // check label is in set
        return !labels.contains(label);
    }
143169
public void addLabel(Label label) {
        assertNotNull(label);
        labels.add(label);
        label.setParent(this);
    }
3005
public void setLabels(INDArray... labels) {
        if (labels != null && labels.length != this.numOutputArrays) {
            throw new IllegalArgumentException("Invalid output array: network has " + numOutputArrays
                    + " outputs, but array is of length " + labels.length);
        }
        this.labels = labels;
    }
1086993
public double score(List<L> guesses, List<L> trueLabels) {
    clearCounts();
    addGuesses(guesses, trueLabels);
    finalizeCounts();
    return getFMeasure();
  }