Custom evaluation metric on CRPS.
497253
def CRPS(label, pred):
    
    for i in range(pred.shape[0]):
        for j in range(pred.shape[1] - 1):
            if pred[i, j] > pred[i, j + 1]:
                pred[i, j + 1] = pred[i, j]
    return np.sum(np.square(label - pred)) / label.size
863663
function report(stream, metrics, thresholds) {
  const target = (!elv(thresholds)) ? -1 : thresholds.target;
  const dimensions = tableDimensions(metrics);
  const border = divider(dimensions);
  let value = header(dimensions);

  for (let i = 0; i < metrics.length; i++) {
    if (i > 0) value += border;
    const metric = metrics[i];
    value += row(metric, dimensions, i === target);
  }

  value += divider(dimensions, 'bottom');
  value += '\n';

  stream.write(value);
}
443262
def get_custom_metrics():
    
    _rse = mx.metric.create(rse)
    _rae = mx.metric.create(rae)
    _corr = mx.metric.create(corr)
    return mx.metric.create([_rae, _rse, _corr])
444086
def _process_metrics(
        self, data, url, services_incl_filter=None, services_excl_filter=None, custom_tags=None, active_tag=None
    ):
        
        hostname = data['svname']
        service_name = data['pxname']
        back_or_front = data['back_or_front']
        custom_tags = [] if custom_tags is None else custom_tags
        active_tag = [] if active_tag is None else active_tag
        tags = ["type:%s" % back_or_front, "instance_url:%s" % url, "service:%s" % service_name]
        tags.extend(custom_tags)
        tags.extend(active_tag)

        if self._is_service_excl_filtered(service_name, services_incl_filter, services_excl_filter):
            return

        if back_or_front == Services.BACKEND:
            tags.append('backend:%s' % hostname)
            if data.get('addr'):
                tags.append('server_address:{}'.format(data.get('addr')))

        for key, value in data.items():
            if HAProxy.METRICS.get(key):
                suffix = HAProxy.METRICS[key][1]
                name = "haproxy.%s.%s" % (back_or_front.lower(), suffix)
                try:
                    if HAProxy.METRICS[key][0] == 'rate':
                        self.rate(name, float(value), tags=tags)
                    else:
                        self.gauge(name, float(value), tags=tags)
                except ValueError:
                    pass
129848
def printComparison(results, class_or_prop):
	

	data = []

	Row = namedtuple('Row',[class_or_prop,'VALIDATED'])

	for k,v in sorted(results.items(), key=lambda x: x[1]):
		data += [Row(k, str(v))]

	pprinttable(data)
1268450
def stats_evaluation(stats):
    """"""
    statement = stats.get('statement')
    error = stats.get('error', 0)
    warning = stats.get('warning', 0)
    refactor = stats.get('refactor', 0)
    convention = stats.get('convention', 0)

    if not statement or statement <= 0:
        return None

    malus = float(5 * error + warning + refactor + convention)
    malus_ratio = malus / statement
    return 10.0 - (malus_ratio * 10)
582433
def update_metric(self, eval_metric, labels, pre_sliced=False):
        
        assert self.binded and self.params_initialized
        self._curr_module.update_metric(eval_metric, labels, pre_sliced)
678502
def evaluate_single_config(
    hparams, sampling_temp, max_num_noops, agent_model_dir,
    eval_fn=_eval_fn_with_learner
):
  """"""
  tf.logging.info("Evaluating metric %s", get_metric_name(
      sampling_temp, max_num_noops, clipped=False
  ))
  eval_hparams = trainer_lib.create_hparams(hparams.base_algo_params)
  env = setup_env(
      hparams, batch_size=hparams.eval_batch_size, max_num_noops=max_num_noops,
      rl_env_max_episode_steps=hparams.eval_rl_env_max_episode_steps,
      env_name=hparams.rl_env_name)
  env.start_new_epoch(0)
  eval_fn(env, hparams, eval_hparams, agent_model_dir, sampling_temp)
  rollouts = env.current_epoch_rollouts()
  env.close()

  return tuple(
      compute_mean_reward(rollouts, clipped) for clipped in (True, False)
  )
399269
private static DecoratorMap metric_map_(Map<IdentifierPair, MetricMatcher> matchers) {
        if (matchers.isEmpty()) return concat();

        return new DecoratorMap() {
            @Override
            public Stream<Consumer<MutableContext>> map(Stream<Consumer<MutableContext>> in, Context ctx) {
                final Map<IdentifierPair, List<Map.Entry<MetricMatcher.MatchedName, MetricValue>>> metricMapping = matchers.entrySet().stream()
                        .collect(Collectors.toMap(Map.Entry::getKey,
                                mmEntry -> mmEntry.getValue().filter(ctx).collect(Collectors.toList())));

                for (final Map.Entry<IdentifierPair, List<Map.Entry<MetricMatcher.MatchedName, MetricValue>>> mm
                     : metricMapping.entrySet()) {
                    final String groupIdentifier = mm.getKey().getGroup();
                    final String metricIdentifier = mm.getKey().getMetric();
                    final List<Consumer<MutableContext>> applications = mm.getValue().stream()
                            .map(matchedMetric -> {
                                Consumer<MutableContext> application = (nestedCtx) -> {
                                    nestedCtx.putGroupAliasByName(groupIdentifier, matchedMetric.getKey().getMatchedGroup()::getGroup);
                                    nestedCtx.putMetricAliasByName(metricIdentifier, matchedMetric.getKey().getMatchedGroup()::getGroup, matchedMetric.getKey()::getMetric);
                                };
                                return application;
                            })
                            .collect(Collectors.toList());

                    in = in.flatMap(inApplication -> applications.stream().map(inApplication::andThen));
                }
                return in;
            }
        };
    }
89593
def evaluate_perceptron(ctx, model, corpus):
    """"""
    click.echo('chemdataextractor.pos.evaluate')
    if corpus == 'wsj':
        evaluation = wsj_evaluation
        sents = list(evaluation.tagged_sents())
        for i, wsj_sent in enumerate(sents):
            sents[i] = [t for t in wsj_sent if not t[1] == u'-NONE-']
    elif corpus == 'genia':
        evaluation = genia_evaluation
        sents = list(evaluation.tagged_sents())
        # Translate GENIA bracket tags
        for i, genia_sent in enumerate(sents):
            for j, (token, tag) in enumerate(genia_sent):
                if tag == u'(':
                    sents[i][j] = (token, u'-LRB-')
                elif tag == u')':
                    sents[i][j] = (token, u'-RRB-')
    else:
        raise click.ClickException('Invalid corpus')
    tagger = ChemApPosTagger(model=model)
    accuracy = tagger.evaluate(sents)
    click.echo('%s on %s: %s' % (model, evaluation, accuracy))
695775
def create_eager_metrics_for_problem(problem, model_hparams):
  """"""
  metric_fns = problem.eval_metric_fns(model_hparams)
  problem_hparams = problem.get_hparams(model_hparams)
  target_modality = problem_hparams.modality["targets"]
  weights_fn = model_hparams.weights_fn.get(
      "targets",
      modalities.get_weights_fn(target_modality))
  return create_eager_metrics_internal(metric_fns, weights_fn=weights_fn)
1155798
def _variant_eval_metrics(self, dup_bam):
        
        base, ext = os.path.splitext(dup_bam)
        end_strip = "-dup"
        base = base[:-len(end_strip)] if base.endswith(end_strip) else base
        mfiles = glob.glob("%s*eval_metrics" % base)
        if len(mfiles) > 0:
            with open(mfiles[0]) as in_handle:
                # pull the metrics as JSON from the last line in the file
                for line in in_handle:
                    pass
                metrics = json.loads(line)
            return metrics
        else:
            return None
528921
def submit_as_gauge_and_monotonic_count(self, metric_suffix, metric, scraper_config):
        
        metric_name = scraper_config['namespace'] + metric_suffix
        for sample in metric.samples:
            # Explicit shallow copy of the instance tags
            _tags = list(scraper_config['custom_tags'])

            for label_name, label_value in iteritems(sample[self.SAMPLE_LABELS]):
                _tags.append('{}:{}'.format(label_name, label_value))
            # submit raw metric
            self.gauge(metric_name, sample[self.SAMPLE_VALUE], _tags)
            # submit rate metric
            self.monotonic_count(metric_name + '.count', sample[self.SAMPLE_VALUE], _tags)
941192
private List<Metric> analyzeResponse(final ICommandLine opt, final String response, final int elapsed) throws MetricGatheringException {
        List<Metric> metrics = new ArrayList<Metric>();
        metrics.add(new Metric("time", "", new BigDecimal(elapsed), null, null));

        if (!opt.hasOption("certificate")) {
            if (opt.hasOption("string")) {
                boolean found = false;
                String string = opt.getOptionValue("string");
                found = response.contains(string);
                metrics.add(new Metric("string", "", new BigDecimal(Utils.getIntValue(found)), null, null));
            }
            if (opt.hasOption("expect")) {
                int count = 0;
                String[] values = opt.getOptionValue("expect").split(",");
                for (String value : values) {
                    if (response.contains(value)) {
                        count++;
                    }
                }
                metrics.add(new Metric("expect", String.valueOf(count) + " times. ", new BigDecimal(count), null, null));
            }
            if (opt.hasOption("regex")) {
                String regex = opt.getOptionValue("regex");
                Pattern p = null;
                int flags = 0;
                if (opt.hasOption("eregi")) {
                    flags = Pattern.CASE_INSENSITIVE;
                }
                if (opt.hasOption("linespan")) {
                    flags = flags | Pattern.MULTILINE;
                }
                p = Pattern.compile(regex, flags);
                boolean found = p.matcher(response).find();
                if (opt.hasOption("invert-regex")) {
                    metrics.add(new Metric("invert-regex", String.valueOf(found), new BigDecimal(Utils.getIntValue(found)), null, null));
                } else {
                    metrics.add(new Metric("regex", String.valueOf(found), new BigDecimal(Utils.getIntValue(found)), null, null));
                }
            }
        }
        return metrics;
    }
586569
def train_crf(ctx, input, output, clusters):
    """"""
    click.echo('chemdataextractor.crf.train')
    sentences = []
    for line in input:
        sentence = []
        for t in line.split():
            token, tag, iob = t.rsplit('/', 2)
            sentence.append(((token, tag), iob))
        if sentence:
            sentences.append(sentence)

    tagger = CrfCemTagger(clusters=clusters)
    tagger.train(sentences, output)
686465
def evaluate_all_configs(
    hparams, agent_model_dir, eval_fn=_eval_fn_with_learner
):
  """"""
  metrics = {}
  # Iterate over all combinations of sampling temperatures and whether to do
  # initial no-ops.
  for sampling_temp in hparams.eval_sampling_temps:
    # Iterate over a set so if eval_max_num_noops == 0 then it's 1 iteration.
    for max_num_noops in set([hparams.eval_max_num_noops, 0]):
      scores = evaluate_single_config(
          hparams, sampling_temp, max_num_noops, agent_model_dir, eval_fn
      )
      for (score, clipped) in zip(scores, (True, False)):
        metric_name = get_metric_name(sampling_temp, max_num_noops, clipped)
        metrics[metric_name] = score

  return metrics
759802
def evaluate(ctx, model, corpus, clusters):
    """"""
    click.echo('chemdataextractor.pos.evaluate')
    if corpus == 'wsj':
        evaluation = wsj_evaluation
        sents = list(evaluation.tagged_sents())
        for i, wsj_sent in enumerate(sents):
            sents[i] = [t for t in wsj_sent if not t[1] == '-NONE-']
    elif corpus == 'genia':
        evaluation = genia_evaluation
        sents = list(evaluation.tagged_sents())
        # Translate GENIA bracket tags
        for i, genia_sent in enumerate(sents):
            for j, (token, tag) in enumerate(genia_sent):
                if tag == '(':
                    sents[i][j] = (token, '-LRB-')
                elif tag == ')':
                    sents[i][j] = (token, '-RRB-')
    else:
        raise click.ClickException('Invalid corpus')
    tagger = ChemCrfPosTagger(model=model, clusters=clusters)
    accuracy = tagger.evaluate(sents)
    click.echo('%s on %s: %s' % (model, evaluation, accuracy))