Evaluate the model on validation dataset.
486773
def evaluate(dataloader_eval, metric):
    
    metric.reset()
    for _, seqs in enumerate(dataloader_eval):
        input_ids, valid_len, type_ids, label = seqs
        out = model(
            input_ids.as_in_context(ctx), type_ids.as_in_context(ctx),
            valid_len.astype('float32').as_in_context(ctx))
        metric.update([label], [out])
    metric_nm, metric_val = metric.get()
    if not isinstance(metric_nm, list):
        metric_nm = [metric_nm]
        metric_val = [metric_val]
    metric_str = 'validation metrics:' + ','.join(
        [i + ':%.4f' for i in metric_nm])
    logging.info(metric_str, *metric_val)
1004111
def evaluate():
    
    log.info('Loader dev data...')
    if version_2:
        dev_data = SQuAD('dev', version='2.0')
    else:
        dev_data = SQuAD('dev', version='1.1')
    log.info('Number of records in Train data:{}'.format(len(dev_data)))

    dev_dataset = dev_data.transform(
        SQuADTransform(
            berttoken,
            max_seq_length=max_seq_length,
            doc_stride=doc_stride,
            max_query_length=max_query_length,
            is_pad=True,
            is_training=False)._transform)

    dev_data_transform, _ = preprocess_dataset(
        dev_data, SQuADTransform(
            berttoken,
            max_seq_length=max_seq_length,
            doc_stride=doc_stride,
            max_query_length=max_query_length,
            is_pad=True,
            is_training=False))
    log.info('The number of examples after preprocessing:{}'.format(
        len(dev_data_transform)))

    dev_dataloader = mx.gluon.data.DataLoader(
        dev_data_transform,
        batchify_fn=batchify_fn,
        num_workers=4, batch_size=test_batch_size, shuffle=False, last_batch='keep')

    log.info('Start predict')

    _Result = collections.namedtuple(
        '_Result', ['example_id', 'start_logits', 'end_logits'])
    all_results = {}

    epoch_tic = time.time()
    total_num = 0
    for data in dev_dataloader:
        example_ids, inputs, token_types, valid_length, _, _ = data
        total_num += len(inputs)
        out = net(inputs.astype('float32').as_in_context(ctx),
                  token_types.astype('float32').as_in_context(ctx),
                  valid_length.astype('float32').as_in_context(ctx))

        output = nd.split(out, axis=2, num_outputs=2)
        start_logits = output[0].reshape((0, -3)).asnumpy()
        end_logits = output[1].reshape((0, -3)).asnumpy()

        for example_id, start, end in zip(example_ids, start_logits, end_logits):
            example_id = example_id.asscalar()
            if example_id not in all_results:
                all_results[example_id] = []
            all_results[example_id].append(
                _Result(example_id, start.tolist(), end.tolist()))
    epoch_toc = time.time()
    log.info('Inference time cost={:.2f} s, Thoughput={:.2f} samples/s'
             .format(epoch_toc - epoch_tic,
                     len(dev_dataloader) / (epoch_toc - epoch_tic)))
    log.info('Get prediction results...')

    all_predictions, all_nbest_json, scores_diff_json = predictions(
        dev_dataset=dev_dataset,
        all_results=all_results,
        tokenizer=nlp.data.BERTBasicTokenizer(lower=lower),
        max_answer_length=max_answer_length,
        null_score_diff_threshold=null_score_diff_threshold,
        n_best_size=n_best_size,
        version_2=version_2)

    with open(os.path.join(output_dir, 'predictions.json'),
              'w', encoding='utf-8') as all_predictions_write:
        all_predictions_write.write(json.dumps(all_predictions))

    with open(os.path.join(output_dir, 'nbest_predictions.json'),
              'w', encoding='utf-8') as all_predictions_write:
        all_predictions_write.write(json.dumps(all_nbest_json))

    if version_2:
        with open(os.path.join(output_dir, 'null_odds.json'),
                  'w', encoding='utf-8') as all_predictions_write:
            all_predictions_write.write(json.dumps(scores_diff_json))
    else:
        log.info(get_F1_EM(dev_data, all_predictions))
589430
def evaluate(
    model,
    data_path,
    gpu_id=-1,
    gold_preproc=False,
    displacy_path=None,
    displacy_limit=25,
    return_scores=False,
):
    
    msg = Printer()
    util.fix_random_seed()
    if gpu_id >= 0:
        util.use_gpu(gpu_id)
    util.set_env_log(False)
    data_path = util.ensure_path(data_path)
    displacy_path = util.ensure_path(displacy_path)
    if not data_path.exists():
        msg.fail("Evaluation data not found", data_path, exits=1)
    if displacy_path and not displacy_path.exists():
        msg.fail("Visualization output directory not found", displacy_path, exits=1)
    corpus = GoldCorpus(data_path, data_path)
    nlp = util.load_model(model)
    dev_docs = list(corpus.dev_docs(nlp, gold_preproc=gold_preproc))
    begin = timer()
    scorer = nlp.evaluate(dev_docs, verbose=False)
    end = timer()
    nwords = sum(len(doc_gold[0]) for doc_gold in dev_docs)
    results = {
        "Time": "%.2f s" % (end - begin),
        "Words": nwords,
        "Words/s": "%.0f" % (nwords / (end - begin)),
        "TOK": "%.2f" % scorer.token_acc,
        "POS": "%.2f" % scorer.tags_acc,
        "UAS": "%.2f" % scorer.uas,
        "LAS": "%.2f" % scorer.las,
        "NER P": "%.2f" % scorer.ents_p,
        "NER R": "%.2f" % scorer.ents_r,
        "NER F": "%.2f" % scorer.ents_f,
    }
    msg.table(results, title="Results")

    if displacy_path:
        docs, golds = zip(*dev_docs)
        render_deps = "parser" in nlp.meta.get("pipeline", [])
        render_ents = "ner" in nlp.meta.get("pipeline", [])
        render_parses(
            docs,
            displacy_path,
            model_name=model,
            limit=displacy_limit,
            deps=render_deps,
            ents=render_ents,
        )
        msg.good("Generated {} parses as HTML".format(displacy_limit), displacy_path)
    if return_scores:
        return scorer.scores
1292038
def _dataset_qa(self, dataset):
        """"""

        # 1. VALIDACIONES
        # chequea que haya por lo menos algún formato de datos reconocido
        has_data_format = helpers.dataset_has_data_distributions(dataset)

        # chequea que algunos campos tengan longitudes mínimas
        has_title = "title" in dataset
        has_description = "description" in dataset
        if has_title:
            has_min_title = len(dataset["title"]) >= MIN_DATASET_TITLE
        else:
            has_min_title = False
        if has_description:
            has_min_desc = len(
                dataset["description"]) >= MIN_DATASET_DESCRIPTION
        else:
            has_min_desc = False

        # 2. EVALUACION DE COSECHA: evalua si se cosecha o no el dataset
        harvest = (has_title and has_description and
                   has_data_format and has_min_title and has_min_desc)

        # 3. NOTAS: genera notas de validación
        notes = []
        if not has_data_format:
            notes.append("No tiene distribuciones con datos.")
        if not has_title:
            notes.append("Dataset sin titulo {}".format(dataset))
        else:
            if not has_min_title:
                notes.append("Titulo tiene menos de {} caracteres".format(
                    MIN_DATASET_TITLE))
        if not has_description:
            notes.append("Dataset sin descripcion {}".format(dataset))
        else:
            if not has_min_desc:
                notes.append("Descripcion tiene menos de {} caracteres".format(
                    MIN_DATASET_DESCRIPTION))

        return harvest, notes
1105105
def evaluate_report(report):
        """"""
        if report["valid"]:
            return
        for warn in report["warnings"]:
            LOGGER.warning(warn)
        # We only ever test one table at a time.
        for err in report["tables"][0]["errors"]:
            LOGGER.error(err["message"])
        raise ValueError("Invalid data file. Please see errors above.")
418537
def validate(self, value, model_instance):
        
        return super(self.__class__, self).validate(value.value, model_instance)
321189
function(set) {
        var dtype, valids, typeClass, self = this;
        //
        if (utils.isString(set)) {
            // resolve data type
            typeClass = this.dataTypes.get(set);
            if (typeClass) { dtype = new typeClass(); }
        }
        else if (set) {
            utils.each(set, function(key, val) {
                if (!utils.hasOwn(set, key)) {
                    return;
                }
                if (key.toLowerCase() === 'type') {
                    // resolve data type
                    typeClass = self.dataTypes.get(val);
                    if (typeClass) { dtype = new typeClass(); }
                } else {
                    // resolve validators
                    typeClass = self.validators.get(key);
                    if (typeClass) {
                        if (!valids) { valids = []; }
                        valids.push(new typeClass(set[key]));
                    }
                }
            });
        }
        return new modellingMetas(dtype, valids);
    }
323760
def _validate_dataset_names(dataset):
    """"""
    def check_name(name):
        if isinstance(name, str):
            if not name:
                raise ValueError('Invalid name for DataArray or Dataset key: '
                                 'string must be length 1 or greater for '
                                 'serialization to netCDF files')
        elif name is not None:
            raise TypeError('DataArray.name or Dataset key must be either a '
                            'string or None for serialization to netCDF files')

    for k in dataset.variables:
        check_name(k)
89593
def evaluate_perceptron(ctx, model, corpus):
    """"""
    click.echo('chemdataextractor.pos.evaluate')
    if corpus == 'wsj':
        evaluation = wsj_evaluation
        sents = list(evaluation.tagged_sents())
        for i, wsj_sent in enumerate(sents):
            sents[i] = [t for t in wsj_sent if not t[1] == u'-NONE-']
    elif corpus == 'genia':
        evaluation = genia_evaluation
        sents = list(evaluation.tagged_sents())
        # Translate GENIA bracket tags
        for i, genia_sent in enumerate(sents):
            for j, (token, tag) in enumerate(genia_sent):
                if tag == u'(':
                    sents[i][j] = (token, u'-LRB-')
                elif tag == u')':
                    sents[i][j] = (token, u'-RRB-')
    else:
        raise click.ClickException('Invalid corpus')
    tagger = ChemApPosTagger(model=model)
    accuracy = tagger.evaluate(sents)
    click.echo('%s on %s: %s' % (model, evaluation, accuracy))
345756
def validate_data_files(problem, data_files, min_size):
  """"""
  # Check that all files are present
  data_dir = os.path.split(data_files[0])[0]
  out_filepaths = problem.out_filepaths(data_dir)
  missing_filepaths = set(out_filepaths) - set(data_files)
  if missing_filepaths:
    tf.logging.error("Missing %d data files", len(missing_filepaths))

  # Check that each file is at least 100M
  too_small = []
  for data_file in data_files:
    length = get_length(data_file)
    if length < min_size:
      too_small.append(data_file)
  if too_small:
    tf.logging.error("%d files too small", len(too_small))

  bad_files = too_small + list(missing_filepaths)
  return bad_files
1018710
def _eval_metric(input_, topk, correct_predictions, examples, phase):
  """"""
  my_parameters = {}
  if phase in (Phase.test, Phase.infer):
    dtype = tf.float32
    # Create the variables using tf.Variable because we don't want to share.
    count = tf.Variable(tf.constant(0, dtype=dtype),
                        name='count_%d' % topk,
                        collections=[bookkeeper.GraphKeys.TEST_VARIABLES],
                        trainable=False)
    correct = tf.Variable(tf.constant(0, dtype=dtype),
                          name='correct_%d' % topk,
                          collections=[bookkeeper.GraphKeys.TEST_VARIABLES],
                          trainable=False)
    my_parameters['count'] = count
    my_parameters['correct'] = correct
    with input_.g.device(count.device):
      examples = tf.assign_add(count, examples)
    with input_.g.device(correct.device):
      correct_predictions = tf.assign_add(correct, correct_predictions)
  return correct_predictions, examples, my_parameters
378316
def validate(self, model, checks=[]):
        """"""
        custom = [
            check_partial(gene_id_check, frozenset(g.id for g in model.genes))
        ]
        super(EssentialityExperiment, self).validate(
            model=model, checks=checks + custom)
879198
def evaluate_with_predictions(data_file, predictions):
    '''
    
    '''
    expected_version = '1.1'
    with open(data_file) as dataset_file:
        dataset_json = json.load(dataset_file)
        if dataset_json['version'] != expected_version:
            print('Evaluation expects v-' + expected_version +
                  ', but got dataset with v-' + dataset_json['version'],
                  file=sys.stderr)
        dataset = dataset_json['data']
    result = _evaluate(dataset, predictions)
    return result['exact_match']
1048124
def main(model_folder):
    """"""
    model_description_file = os.path.join(model_folder, "info.yml")

    # Read the model description file
    with open(model_description_file, 'r') as ymlfile:
        model_description = yaml.load(ymlfile)

    # Analyze model
    logging.info(model_description['model'])
    data = {}
    data['training'] = os.path.join(model_folder, "traindata.hdf5")
    data['testing'] = os.path.join(model_folder, "testdata.hdf5")
    data['validating'] = os.path.join(model_folder, "validdata.hdf5")
    train_model(model_folder)
625428
def check_dataset(dataset, mode):
  """"""

  names = [x['name'] for x in dataset.schema]
  types = [x['type'] for x in dataset.schema]
  if mode == 'train':
    if (set(['image_url', 'label']) != set(names) or any(t != 'STRING' for t in types)):
      raise ValueError('Invalid dataset. Expect only "image_url,label" STRING columns.')
  else:
    if (set(['image_url']) != set(names) and set(['image_url', 'label']) != set(names)) or \
            any(t != 'STRING' for t in types):
      raise ValueError('Invalid dataset. Expect only "image_url" or "image_url,label" ' +
                       'STRING columns.')
202758
function(model, options) {
      if (!(model instanceof Backbone.Model)) {
        var attrs = model;
        model = new this.model(attrs, {collection: this});
        if (model.validate && !model._performValidation(attrs, options)) model = false;
      } else if (!model.collection) {
        model.collection = this;
      }
      return model;
    }
868176
public ClassificationMetrics validate(Dataframe testDataset) {
        logger.info("validate()");

        predict(testDataset);

        ClassificationMetrics vm = new ClassificationMetrics(testDataset);

        return vm;
    }