Scorecard model builder: JIT a subclass with the fast version wired in to
'score(row)'
960768
public static ScorecardModel make(final String name, final double initialScore, RuleTable[] rules) {
    // Get the list of features
    String[] colNames = new String[rules.length];
    for( int i=0; i<rules.length; i++ )
      colNames[i] = rules[i]._name;

    // javassist support for rewriting class files
    ClassPool _pool = ClassPool.getDefault();
    try {
      // Make a javassist class in the java hierarchy
      String cname = uniqueClassName(name);
      CtClass scClass = _pool.makeClass(cname);
      CtClass baseClass = _pool.get("water.score.ScorecardModel"); // Full Name Lookup
      scClass.setSuperclass(baseClass);
      // Produce the scoring method(s)
      ScorecardModel scm = new ScorecardModel(name, colNames,initialScore, rules);
      scm.makeScoreHashMethod(scClass);
      scm.makeScoreAryMethod(scClass);
      // Produce a 1-arg constructor
      String cons = "  public "+cname+"(water.score.ScorecardModel base) { super(base); }";
      CtConstructor happyConst = CtNewConstructor.make(cons,scClass);
      scClass.addConstructor(happyConst);

      Class myClass = scClass.toClass(ScorecardModel.class.getClassLoader(), null);
      Constructor<ScorecardModel> co = myClass.getConstructor(ScorecardModel.class);
      ScorecardModel jitted_scm = co.newInstance(scm);
      return jitted_scm;

    } catch( Exception e ) {
      Log.err(Sys.SCORM,"Javassist failed",e);
    }
    return null;
  }
714024
def to_s
      [
        "MethodProfiler results for: #{@name}",
        Hirb::Helpers::Table.render(
          to_a,
          headers: HEADERS.dup,
          fields: FIELDS.dup,
          filters: {
            min: :to_milliseconds,
            max: :to_milliseconds,
            average: :to_milliseconds,
            total_time: :to_milliseconds,
          },
          description: false
        )
      ].join("\n")
    end
732795
def pick(seq, func, maxobj=None):
    """"""
    maxscore = None
    for obj in seq:
        score = func(obj)
        if maxscore is None or maxscore < score:
            (maxscore, maxobj) = (score, obj)
    return maxobj
987619
def score(args):
    
    from jcvi.formats.base import SetFile
    from jcvi.formats.fasta import Fasta

    p = OptionParser(score.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 3:
        sys.exit(not p.print_help())

    blastfile, fastafile, idsfile = args
    ids = SetFile(idsfile)

    blast = Blast(blastfile)
    scores = defaultdict(int)
    for b in blast:
        query = b.query
        subject = b.subject
        if subject not in ids:
            continue
        scores[query] += b.score

    logging.debug("A total of {0} ids loaded.".format(len(ids)))

    f = Fasta(fastafile)
    for s in f.iterkeys_ordered():
        sc = scores.get(s, 0)
        print("\t".join((s, str(sc))))
1312809
def interact_gridsearch_result_images(show_result_func, cfgdict_list,
                                      cfglbl_list, cfgresult_list,
                                      score_list=None, fnum=None, figtitle='',
                                      unpack=False, max_plots=25, verbose=True,
                                      precision=3, scorelbl='score',
                                      onclick_func=None):
    
    assert callable(show_result_func), 'NEED FUNCTION GOT: %r' % (show_result_func,)

    import utool as ut
    import plottool as pt
    from plottool import plot_helpers as ph
    from plottool import interact_helpers as ih
    if verbose:
        print('Plotting gridsearch results figtitle=%r' % (figtitle,))
    if score_list is None:
        score_list = [None] * len(cfgdict_list)
    else:
        # sort by score if available
        sortx_list = ut.list_argsort(score_list, reverse=True)
        score_list = ut.take(score_list, sortx_list)
        cfgdict_list = ut.take(cfgdict_list, sortx_list)
        cfglbl_list = ut.take(cfglbl_list, sortx_list)
        cfgresult_list = ut.take(cfgresult_list, sortx_list)
    # Dont show too many results only the top few
    score_list = ut.listclip(score_list, max_plots)

    # Show the config results
    fig = pt.figure(fnum=fnum)
    # Get plots for each of the resutls
    nRows, nCols = pt.get_square_row_cols(len(score_list), fix=True)
    next_pnum = pt.make_pnum_nextgen(nRows, nCols)
    for cfgdict, cfglbl, cfgresult, score in zip(cfgdict_list, cfglbl_list,
                                                  cfgresult_list,
                                                  score_list):
        if score is not None:
            cfglbl += '\n' + scorelbl + '=' + ut.repr2(score, precision=precision)
        pnum = next_pnum()
        try:
            if unpack:
                show_result_func(*cfgresult, fnum=fnum, pnum=pnum)
            else:
                show_result_func(cfgresult, fnum=fnum, pnum=pnum)
        except Exception as ex:
            if isinstance(cfgresult, tuple):
                #print(ut.repr4(cfgresult))
                print(ut.depth_profile(cfgresult))
                print(ut.list_type_profile(cfgresult))
            ut.printex(ex, 'error showing', keys=['cfgresult', 'fnum', 'pnum'])
            raise
        #pt.imshow(255 * cfgresult, fnum=fnum, pnum=next_pnum(), title=cfglbl)
        ax = pt.gca()
        pt.set_title(cfglbl, ax=ax)  # , size)
        ph.set_plotdat(ax, 'cfgdict', cfgdict)
        ph.set_plotdat(ax, 'cfglbl', cfglbl)
        ph.set_plotdat(ax, 'cfgresult', cfgresult)
    # Define clicked callback
    def on_clicked(event):
        print('\n[pt] clicked gridsearch axes')
        if event is None or event.xdata is None or event.inaxes is None:
            print('out of axes')
            pass
        else:
            ax = event.inaxes
            plotdat_dict = ph.get_plotdat_dict(ax)
            print(ut.repr4(plotdat_dict))
            cfglbl = ph.get_plotdat(ax, 'cfglbl', None)
            cfgdict = ph.get_plotdat(ax, 'cfgdict', {})
            cfgresult = ph.get_plotdat(ax, 'cfgresult', {})
            infostr_list = [
                ('cfglbl = %s' % (cfglbl,)),
                '',
                ('cfgdict = ' + ut.repr4(cfgdict, sorted_=True)),
            ]
            # Call a user defined function if given
            if onclick_func is not None:
                if unpack:
                    onclick_func(*cfgresult)
                else:
                    onclick_func(cfgresult)
            infostr = ut.msgblock('CLICKED', '\n'.join(infostr_list))
            print(infostr)
    # Connect callbacks
    ih.connect_callback(fig, 'button_press_event', on_clicked)
    pt.set_figtitle(figtitle)
1097544
def _from_java(cls, java_stage):
        

        estimator, epms, evaluator = super(TrainValidationSplit, cls)._from_java_impl(java_stage)
        trainRatio = java_stage.getTrainRatio()
        seed = java_stage.getSeed()
        parallelism = java_stage.getParallelism()
        collectSubModels = java_stage.getCollectSubModels()
        # Create a new instance of this stage.
        py_stage = cls(estimator=estimator, estimatorParamMaps=epms, evaluator=evaluator,
                       trainRatio=trainRatio, seed=seed, parallelism=parallelism,
                       collectSubModels=collectSubModels)
        py_stage._resetUid(java_stage.uid())
        return py_stage
844100
@SuppressWarnings("unused") // called through reflection by RequestServer
  public JobV3 predictAsync(int version, final ModelMetricsListSchemaV3 s) {
    // parameters checking:
    if (null == s.model) throw new H2OIllegalArgumentException("model", "predict", s.model);
    if (null == DKV.get(s.model.name)) throw new H2OKeyNotFoundArgumentException("model", "predict", s.model.name);

    if (null == s.frame) throw new H2OIllegalArgumentException("frame", "predict", s.frame);
    if (null == DKV.get(s.frame.name)) throw new H2OKeyNotFoundArgumentException("frame", "predict", s.frame.name);

    if (s.deviances || null != s.deviances_frame) throw new H2OIllegalArgumentException("deviances", "not supported for async", s.deviances_frame);

    final ModelMetricsList parms = s.createAndFillImpl();

    //predict2 does not return modelmetrics, so cannot handle deeplearning: reconstruction_error (anomaly) or GLRM: reconstruct and archetypes
    //predict2 can handle deeplearning: deepfeatures and predict

    if (s.deep_features_hidden_layer > 0 || s.deep_features_hidden_layer_name != null) {
      if (null == parms._predictions_name)
        parms._predictions_name = "deep_features" + Key.make().toString().substring(0, 5) + "_" +
                parms._model._key.toString() + "_on_" + parms._frame._key.toString();
    } else if (null == parms._predictions_name) {
      if (parms._exemplar_index >= 0) {
        parms._predictions_name = "members_" + parms._model._key.toString() + "_for_exemplar_" + parms._exemplar_index;
      } else {
        parms._predictions_name = "predictions" + Key.make().toString().substring(0, 5) + "_" + parms._model._key.toString() + "_on_" + parms._frame._key.toString();
      }
    }

    final Job<Frame> j = new Job(Key.make(parms._predictions_name), Frame.class.getName(), "prediction");


    H2O.H2OCountedCompleter work = new H2O.H2OCountedCompleter() {
      @Override
      public void compute2() {
        if (s.deep_features_hidden_layer < 0 && s.deep_features_hidden_layer_name == null) {
          parms._model.score(parms._frame, parms._predictions_name, j, false, CFuncRef.from(s.custom_metric_func));
        }
        else if (s.deep_features_hidden_layer_name != null){
          Frame predictions = null;
          try {
            predictions = ((Model.DeepFeatures) parms._model).scoreDeepFeatures(parms._frame, s.deep_features_hidden_layer_name, j);
          } catch(IllegalArgumentException e) {
            Log.warn(e.getMessage());
            throw e;
          }
          if (predictions!=null) {
            predictions = new Frame(Key.<Frame>make(parms._predictions_name), predictions.names(), predictions.vecs());
            DKV.put(predictions._key, predictions);
          }
        }
        else {
          Frame predictions = ((Model.DeepFeatures) parms._model).scoreDeepFeatures(parms._frame, s.deep_features_hidden_layer, j);
          predictions = new Frame(Key.<Frame>make(parms._predictions_name), predictions.names(), predictions.vecs());
          DKV.put(predictions._key, predictions);
        }
        if ((parms._model._warningsP != null) && (parms._model._warningsP.length > 0)) { // add prediction warning here only
          String[] allWarnings = (String[]) ArrayUtils.addAll(j.warns(), parms._model._warningsP); // copy both over
          j.setWarnings(allWarnings);
        }
        tryComplete();
      }
    };
    j.start(work, parms._frame.anyVec().nChunks());
    return new JobV3().fillFromImpl(j);
  }
694693
private ScoreWeight getDashboardScoreFromWidgets(List<ScoreWeight> widgetScores) {
    if (null == widgetScores) {
      return getDashboardScore(new ScoreTypeValue(Constants.ZERO_SCORE), widgetScores);
    }

    try {
      //Calculate score
      return getDashboardScore(
        ScoreCalculationUtils.calculateComponentScoreTypeValue(widgetScores, PropagateType.dashboard),
        widgetScores
      );
    } catch (PropagateScoreException ex) {
      ScoreWeight scoreDashboard = new ScoreWeight();
      scoreDashboard.setScore(ex.getScore());
      scoreDashboard.setMessage(ex.getMessage());
      scoreDashboard.setState(ex.getState());
      scoreDashboard.setChildren(widgetScores);
      return scoreDashboard;
    }
  }
1320676
def display_grid_scores(grid_scores, top=None):
    """"""
    grid_scores = sorted(grid_scores, key=lambda x: x[1], reverse=True)
    if top is not None:
        grid_scores = grid_scores[:top]
        
    # Compute a threshold for staring models with overlapping stderr:
    _, best_mean, best_scores = grid_scores[0]
    threshold = best_mean - 2 * sem(best_scores)
    
    for params, mean_score, scores in grid_scores:
        append_star = mean_score + 2 * sem(scores) > threshold
        print(display_scores(params, scores, append_star=append_star))
721872
def _init_map(self, record_types=None, **kwargs):
        """"""
        osid_objects.OsidObjectForm._init_map(self, record_types=record_types)
        self._my_map['nextAssessmentPartId'] = str(kwargs['next_assessment_part_id'])
        self._my_map['cumulative'] = self._cumulative_default
        self._my_map['minimumScore'] = self._minimum_score_default
        self._my_map['maximumScore'] = self._maximum_score_default
        self._my_map['assessmentPartId'] = str(kwargs['assessment_part_id'])
        self._my_map['assignedBankIds'] = [str(kwargs['bank_id'])]
        self._my_map['appliedAssessmentPartIds'] = []
943561
def _gen_indicator_method(self, name, custom_class, value_count):
        
        method_name = name.replace(' ', '_').lower()
        tcex = self.tcex

        # Add Method for each Custom Indicator class
        def method_1(owner, value1, **kwargs):  # pylint: disable=W0641
            """Add Custom Indicator data to Batch object"""
            return custom_class(tcex, value1, owner=owner, **kwargs)

        def method_2(owner, value1, value2, **kwargs):  # pylint: disable=W0641
            """Add Custom Indicator data to Batch object"""
            return custom_class(tcex, value1, value2, owner=owner, **kwargs)

        def method_3(owner, value1, value2, value3, **kwargs):  # pylint: disable=W0641
            """Add Custom Indicator data to Batch object"""
            return custom_class(tcex, value1, value2, value3, owner=owner, **kwargs)

        method = locals()['method_{}'.format(value_count)]
        setattr(self, method_name, method)
415112
def preprocess(df):
    """"""

    def process_row(idx_row):
        idx, row = idx_row

        # press-press latency
        tau = 1000 * row[4::3].astype(float).values
        tau = np.r_[np.median(tau), tau]

        # duration
        duration = 1000 * row[3::3].astype(float).values

        keyname = list('.tie5Roanl') + ['enter']

        return pd.DataFrame.from_items([
            ('user', [row['subject']] * 11),
            ('session', [row['sessionIndex'] * 100 + row['rep']] * 11),
            ('tau', tau),
            ('duration', duration),
            ('event', keyname)
        ])

    df = pd.concat(map(process_row, df.iterrows())).set_index(['user', 'session'])
    return df
806661
private static void insideAlgorithm(final double[][] scores, final ProjTreeChart inChart, boolean singleRoot) {
        final int n = scores.length;        
        final int startIdx = singleRoot ? 1 : 0;         

        // Initialize.
        for (int s = 0; s < n; s++) {
            inChart.setScore(s, s, RIGHT, COMPLETE, 0.0);
            inChart.setScore(s, s, LEFT, COMPLETE, 0.0);
        }
                
        // Parse.
        for (int width = 1; width < n; width++) {
            for (int s = startIdx; s < n - width; s++) {
                int t = s + width;
                
                // First create incomplete items.
                for (int r=s; r<t; r++) {
                    for (int d=0; d<2; d++) {
                        double edgeScore = (d==LEFT) ? scores[t][s] : scores[s][t];
                        double score = inChart.getScore(s, r, RIGHT, COMPLETE) +
                                       inChart.getScore(r+1, t, LEFT, COMPLETE) +  
                                       edgeScore;
                        inChart.updateCell(s, t, d, INCOMPLETE, score, r);
                    }
                }
                
                // Second create complete items.
                // -- Left side.
                for (int r=s; r<t; r++) {
                    final int d = LEFT;
                    double score = inChart.getScore(s, r, d, COMPLETE) +
                                inChart.getScore(r, t, d, INCOMPLETE);
                    inChart.updateCell(s, t, d, COMPLETE, score, r);
                }
                // -- Right side.
                for (int r=s+1; r<=t; r++) {
                    final int d = RIGHT;
                    double score = inChart.getScore(s, r, d, INCOMPLETE) +
                                   inChart.getScore(r, t, d, COMPLETE);
                    inChart.updateCell(s, t, d, COMPLETE, score, r);
                }                
            }
        }
        
        if (singleRoot) {
            // Build goal constituents by combining left and right complete
            // constituents, on the left and right respectively. This corresponds to
            // left and right triangles. (Note: this is the opposite of how we
            // build an incomplete constituent.)
            for (int r=1; r<n; r++) {
                double score = inChart.getScore(1, r, LEFT, COMPLETE) +
                               inChart.getScore(r, n-1, RIGHT, COMPLETE) + 
                               scores[0][r];
                inChart.updateCell(0, r, RIGHT, INCOMPLETE, score, r);
                inChart.updateCell(0, n-1, RIGHT, COMPLETE, score, r);
            }
        }
    }
78771
def set_classifier_interface_params(spec, features, class_labels,
        model_accessor_for_class_labels, output_features = None):
    
    # Normalize the features list.
    features = _fm.process_or_validate_features(features)

    if class_labels is None:
        raise ValueError("List of class labels must be provided.")

    n_classes = len(class_labels)

    output_features = _fm.process_or_validate_classifier_output_features(output_features, class_labels)

    if len(output_features) == 1:
        predicted_class_output, pred_cl_type = output_features[0]
        score_output = None
    elif len(output_features) == 2:
        predicted_class_output, pred_cl_type = output_features[0]
        score_output, score_output_type = output_features[1]
    else:
        raise ValueError("Provided output classes for a classifier must be "
                "a list of features, predicted class and (optionally) class_score.")

    spec.description.predictedFeatureName = predicted_class_output

    # Are they out of order?
    if not (pred_cl_type == datatypes.Int64() or pred_cl_type == datatypes.String()):
        raise ValueError("Provided predicted class output type not Int64 or String (%s)."
                % repr(pred_cl_type))

    if score_output is not None:
        if not isinstance(score_output_type, datatypes.Dictionary):
            raise ValueError("Provided class score output type not a Dictionary (%s)."
                    % repr(score_output_type))

        if score_output_type.key_type != pred_cl_type:
            raise ValueError(("Provided class score output (%s) key_type (%s) does not "
                    "match type of class prediction (%s).")
                    % (score_output, repr(score_output_type.key_type), repr(pred_cl_type)))

        spec.description.predictedProbabilitiesName = score_output

    # add input
    for index, (cur_input_name, input_type) in enumerate(features):
        input_ = spec.description.input.add()
        input_.name = cur_input_name
        datatypes._set_datatype(input_.type, input_type)

    # add output
    for index, (cur_output_name, output_type) in enumerate(output_features):
        output_ = spec.description.output.add()
        output_.name = cur_output_name
        datatypes._set_datatype(output_.type, output_type)

    # Worry about the class labels
    if pred_cl_type == datatypes.String():
        try:
            for c in class_labels:
                getattr(spec, model_accessor_for_class_labels).stringClassLabels.vector.append(str(c))
        # Not all the classifiers have class labels; in particular the pipeline
        # classifier.  Thus it's not an error if we can't actually set them.
        except AttributeError:
            pass

    else:
        for c in class_labels:
            conv_error = False
            try:
                if not (int(c) == c):
                    conv_error = True
            except:
                conv_error = True

            if conv_error:
                raise TypeError(("Cannot cast '%s' class to an int type " % str(c))
                + "(class type determined by type of first class).")

            try:
                getattr(spec, model_accessor_for_class_labels).int64ClassLabels.vector.append(int(c))
            # Not all the classifiers have class labels; in particular the pipeline
            # classifier.  Thus it's not an error if we can't actually set them.
            except AttributeError:
                break

    # And we are done!
    return spec
447679
def _to_ijv(ss, row_levels=(0, ), column_levels=(1, ), sort_labels=False):
    
    # index and column levels must be a partition of the index
    _check_is_partition([row_levels, column_levels], range(ss.index.nlevels))

    # from the SparseSeries: get the labels and data for non-null entries
    values = ss._data.internal_values()._valid_sp_values

    nonnull_labels = ss.dropna()

    def get_indexers(levels):
        """ Return sparse coords and dense labels for subset levels """

        # TODO: how to do this better? cleanly slice nonnull_labels given the
        # coord
        values_ilabels = [tuple(x[i] for i in levels)
                          for x in nonnull_labels.index]
        if len(levels) == 1:
            values_ilabels = [x[0] for x in values_ilabels]

        # # performance issues with groupby ###################################
        # TODO: these two lines can rejplace the code below but
        # groupby is too slow (in some cases at least)
        # labels_to_i = ss.groupby(level=levels, sort=sort_labels).first()
        # labels_to_i[:] = np.arange(labels_to_i.shape[0])

        def _get_label_to_i_dict(labels, sort_labels=False):
            """ Return OrderedDict of unique labels to number.
            Optionally sort by label.
            """
            labels = Index(lmap(tuple, labels)).unique().tolist()  # squish
            if sort_labels:
                labels = sorted(list(labels))
            d = OrderedDict((k, i) for i, k in enumerate(labels))
            return (d)

        def _get_index_subset_to_coord_dict(index, subset, sort_labels=False):
            ilabels = list(zip(*[index._get_level_values(i) for i in subset]))
            labels_to_i = _get_label_to_i_dict(ilabels,
                                               sort_labels=sort_labels)
            labels_to_i = Series(labels_to_i)
            if len(subset) > 1:
                labels_to_i.index = MultiIndex.from_tuples(labels_to_i.index)
                labels_to_i.index.names = [index.names[i] for i in subset]
            else:
                labels_to_i.index = Index(x[0] for x in labels_to_i.index)
                labels_to_i.index.name = index.names[subset[0]]

            labels_to_i.name = 'value'
            return (labels_to_i)

        labels_to_i = _get_index_subset_to_coord_dict(ss.index, levels,
                                                      sort_labels=sort_labels)
        # #####################################################################
        # #####################################################################

        i_coord = labels_to_i[values_ilabels].tolist()
        i_labels = labels_to_i.index.tolist()

        return i_coord, i_labels

    i_coord, i_labels = get_indexers(row_levels)
    j_coord, j_labels = get_indexers(column_levels)

    return values, i_coord, j_coord, i_labels, j_labels
1332935
def _to_java(self):
        

        estimator, epms, evaluator = super(CrossValidator, self)._to_java_impl()

        _java_obj = JavaParams._new_java_obj("org.apache.spark.ml.tuning.CrossValidator", self.uid)
        _java_obj.setEstimatorParamMaps(epms)
        _java_obj.setEvaluator(evaluator)
        _java_obj.setEstimator(estimator)
        _java_obj.setSeed(self.getSeed())
        _java_obj.setNumFolds(self.getNumFolds())
        _java_obj.setParallelism(self.getParallelism())
        _java_obj.setCollectSubModels(self.getCollectSubModels())

        return _java_obj
549012
def _process_trait_mappings(self, raw, limit=None):
        
        if self.test_mode:
            graph = self.testgraph
        else:
            graph = self.graph
        line_counter = 0
        model = Model(graph)

        with open(raw, 'r') as csvfile:
            filereader = csv.reader(csvfile, delimiter=',', quotechar='\"')
            next(filereader, None)  # skip header line
            for row in filereader:
                line_counter += 1
                # need to skip the last line
                if len(row) < 8:
                    LOG.info("skipping line %d: %s", line_counter, '\t'.join(row))
                    continue
                (vto_id, pto_id, cmo_id, ato_column, species, trait_class,
                 trait_type, qtl_count) = row

                ato_id = re.sub(
                    r'ATO #', 'AQTLTrait:', re.sub(
                        r'\].*', '', re.sub(r'\[', '', ato_column)))
                ato_id = ato_id.strip()

                ato_label = re.sub(r'.*\]\s*', '', ato_column)

                model.addClassToGraph(ato_id, ato_label.strip())

                if re.match(r'VT:.*', vto_id):
                    model.addClassToGraph(vto_id, None)
                    model.addEquivalentClass(ato_id, vto_id)
                if re.match(r'LPT:.*', pto_id):
                    model.addClassToGraph(pto_id, None)
                    model.addXref(ato_id, pto_id)
                if re.match(r'CMO:.*', cmo_id):
                    model.addClassToGraph(cmo_id, None)
                    model.addXref(ato_id, cmo_id)

        LOG.info("Done with trait mappings")
        return