make_inst: prepare data for the diet model
320538
def make_inst():
    """"""
    F,c,d = multidict({       # cost # composition
        "QPounder" :  [ 1.84, {"Cal":510, "Carbo":34, "Protein":28,
                               "VitA":15, "VitC":  6, "Calc":30, "Iron":20}],
        "McLean"   :  [ 2.19, {"Cal":370, "Carbo":35, "Protein":24, "VitA":15,
                               "VitC": 10, "Calc":20, "Iron":20}],
        "Big Mac"  :  [ 1.84, {"Cal":500, "Carbo":42, "Protein":25,
                               "VitA": 6, "VitC":  2, "Calc":25, "Iron":20}],
        "FFilet"   :  [ 1.44, {"Cal":370, "Carbo":38, "Protein":14,
                               "VitA": 2, "VitC":  0, "Calc":15, "Iron":10}],
        "Chicken"  :  [ 2.29, {"Cal":400, "Carbo":42, "Protein":31,
                               "VitA": 8, "VitC": 15, "Calc":15, "Iron": 8}],
        "Fries"    :  [  .77, {"Cal":220, "Carbo":26, "Protein": 3,
                               "VitA": 0, "VitC": 15, "Calc": 0, "Iron": 2}],
        "McMuffin" :  [ 1.29, {"Cal":345, "Carbo":27, "Protein":15,
                               "VitA": 4, "VitC":  0, "Calc":20, "Iron":15}],
        "1% LFMilk":  [  .60, {"Cal":110, "Carbo":12, "Protein": 9,
                               "VitA":10, "VitC":  4, "Calc":30, "Iron": 0}],
        "OrgJuice" :  [  .72, {"Cal": 80, "Carbo":20, "Protein": 1,
                               "VitA": 2, "VitC":120, "Calc": 2, "Iron": 2}],
        })

    N,a,b = multidict({       # min,max intake
        "Cal"     : [ 2000,  None ],
        "Carbo"   : [  350,  375 ],
        "Protein" : [   55,  None ],
        "VitA"    : [  100,  None ],
        "VitC"    : [  100,  None ],
        "Calc"    : [  100,  None ],
        "Iron"    : [  100,  None ],
     })

    return F,N,a,b,c,d
591925
def build_model(self, n_features, regtype='none'):
        
        self._create_placeholders(n_features)
        self._create_variables(n_features)
        self.encode = self.sample_hidden_from_visible(self.input_data)[0]
        self.reconstruction = self.sample_visible_from_hidden(
            self.encode, n_features)

        hprob0, hstate0, vprob, hprob1, hstate1 = self.gibbs_sampling_step(
            self.input_data, n_features)
        positive = self.compute_positive_association(self.input_data,
                                                     hprob0, hstate0)

        nn_input = vprob

        for step in range(self.gibbs_sampling_steps - 1):
            hprob, hstate, vprob, hprob1, hstate1 = self.gibbs_sampling_step(
                nn_input, n_features)
            nn_input = vprob

        negative = tf.matmul(tf.transpose(vprob), hprob1)

        self.w_upd8 = self.W.assign_add(
            self.learning_rate * (positive - negative) / self.batch_size)

        self.bh_upd8 = self.bh_.assign_add(tf.multiply(self.learning_rate, tf.reduce_mean(
            tf.subtract(hprob0, hprob1), 0)))

        self.bv_upd8 = self.bv_.assign_add(tf.multiply(self.learning_rate, tf.reduce_mean(
            tf.subtract(self.input_data, vprob), 0)))

        variables = [self.W, self.bh_, self.bv_]
        regterm = Layers.regularization(variables, self.regtype, self.regcoef)

        self.cost = self.loss.compile(vprob, self.input_data, regterm=regterm)
515246
def make_funcs(dataset, setdir, store):
    """"""
    return {
        'cat': lambda *lists: [x for lst in lists for x in lst],
        'comments': lambda: None,
        'detail_route': detail_route,
        'format': lambda fmt, *args: fmt.format(*args),
        'get': partial(getnode, dataset, setdir, store),
        'join': lambda sep, *args: sep.join([x for x in args if x]),
        'len': len,
        'link': (lambda href, title, target=None:
                 {'href': href or "",
                  'title': title or "",
                  'target': '_blank' if target is None else target}),
        'list': lambda *x: filter(None, list(x)),
        'max': max,
        'min': min,
        'status': lambda: ['#STATUS#'],
        'sum': sum,
        'tags': lambda: ['#TAGS#'],
        'trace': print_trace,
    }
352592
def dM(self, k, t, param, Mkt, tips=None, gaps=None):
        """"""
        assert 0 <= k < self.ncats
        assert ((param in self.freeparams) or (param == 't') or (
                param == self.distributedparam))
        assert param not in self.distributionparams
        return self._models[k].dM(t, param, Mkt, tips=tips, gaps=gaps)
658604
def prepare(data):
    """"""
    result = {}
    result["mode"] = data.get("mode")
    result["path"] = data.get("path")
    result["type"] = data.get("type")
    result["sha"] = data.get("sha")
    return result
944131
def builddata(self, mopt=None, data=None, pdata=None, prior=None):
        
        if pdata is None:
            if data is None:
                raise ValueError('no data or pdata')
            pdata = gvar.BufferDict()
            for m in self.flatmodels:
                pdata[m.datatag] = (
                    m.builddata(data) if m.ncg <= 1 else
                    MultiFitter.coarse_grain(m.builddata(data), m.ncg)
                    )
        else:
            npdata = gvar.BufferDict()
            for m in self.flatmodels:
                npdata[m.datatag] = pdata[m.datatag]
            pdata = npdata
        if mopt is not None:
            fitfcn = self.buildfitfcn()
            p_all = self.buildprior(prior=prior, mopt=None)
            f_all = fitfcn(p_all)

            # fcn with part we want to keep
            p_trunc = self.buildprior(prior=prior, mopt=mopt)
            f_trunc = fitfcn(p_trunc)

            # correct pdata
            pdata = gvar.BufferDict(pdata)
            if not self.ratio:
                for m in self.flatmodels:
                    pdata[m.datatag] += f_trunc[m.datatag] - f_all[m.datatag]
            else:
                for m in self.flatmodels:
                    ii = (gvar.mean(f_all[m.datatag]) != 0)
                    ratio = f_trunc[m.datatag][ii] / f_all[m.datatag][ii]
                    pdata[m.datatag][ii] *= ratio
        return pdata
965580
public void trainOnInstanceImpl(Instance inst) {
		checkRoot();

		examplesSeen += inst.weight();
		sumOfValues += inst.weight() * inst.classValue();
		sumOfSquares += inst.weight() * inst.classValue() * inst.classValue();

		for (int i = 0; i < inst.numAttributes() - 1; i++) {
			int aIndex = modelAttIndexToInstanceAttIndex(i, inst);
			sumOfAttrValues.addToValue(i, inst.weight() * inst.value(aIndex));
			sumOfAttrSquares.addToValue(i, inst.weight() * inst.value(aIndex) * inst.value(aIndex));
		}

		double prediction = treeRoot.getPrediction(inst);
		processInstance(inst, treeRoot, prediction, getNormalizedError(inst, prediction), true, false);
	}
329670
def prepare_lazy_data(content, functions_mapping=None, check_variables_set=None, cached=False):
    
    # TODO: refactor type check
    if content is None or isinstance(content, (numeric_types, bool, type)):
        return content

    elif isinstance(content, (list, set, tuple)):
        return [
            prepare_lazy_data(
                item,
                functions_mapping,
                check_variables_set,
                cached
            )
            for item in content
        ]

    elif isinstance(content, dict):
        parsed_content = {}
        for key, value in content.items():
            parsed_key = prepare_lazy_data(
                key,
                functions_mapping,
                check_variables_set,
                cached
            )
            parsed_value = prepare_lazy_data(
                value,
                functions_mapping,
                check_variables_set,
                cached
            )
            parsed_content[parsed_key] = parsed_value

        return parsed_content

    elif isinstance(content, basestring):
        # content is in string format here
        if not is_var_or_func_exist(content):
            # content is neither variable nor function
            # replace $$ notation with $ and consider it as normal char.
            # e.g. abc => abc, abc$$def => abc$def, abc$$$$def$$h => abc$$def$h
            return content.replace("$$", "$")

        functions_mapping = functions_mapping or {}
        check_variables_set = check_variables_set or set()
        content = content.strip()
        content = LazyString(content, functions_mapping, check_variables_set, cached)

    return content
631254
def prepare_targets(options)
      targets = [options[:target], options[:targets]].flatten.compact
      if targets.empty?
        puts 'You must supply the targets you are going to run in the options'
        puts 'passed to regression_manager.run.'
        fail
      end
      Origen.target.loop(options) { |_options| }
      targets
    end
647199
def init_nn_params(scale, layer_sizes, rs=npr.RandomState(0)):
    """"""
    return [(rs.randn(insize, outsize) * scale,   # weight matrix
             rs.randn(outsize) * scale)           # bias vector
            for insize, outsize in zip(layer_sizes[:-1], layer_sizes[1:])]
504127
def fit(args, network, data_loader, eval_metrics=None, batch_end_callback=None):
    """"""
    # kvstore
    kv = mx.kvstore.create(args.kv_store)

    # logging
    head = '%(asctime)-15s Node[' + str(kv.rank) + '] %(message)s'
    if 'log_file' in args and args.log_file is not None:
        log_file = args.log_file
        log_dir = args.log_dir
        log_file_full_name = os.path.join(log_dir, log_file)
        if not os.path.exists(log_dir):
            os.mkdir(log_dir)
        logger = logging.getLogger()
        handler = logging.FileHandler(log_file_full_name)
        formatter = logging.Formatter(head)
        handler.setFormatter(formatter)
        logger.addHandler(handler)
        logger.setLevel(logging.DEBUG)
        logger.info('start with arguments %s', args)
    else:
        logging.basicConfig(level=logging.DEBUG, format=head)
        logging.info('start with arguments %s', args)

    # load model
    model_prefix = args.model_prefix
    if model_prefix is not None:
        model_prefix += "-%d" % (kv.rank)
    model_args = {}
    if args.load_epoch is not None:
        assert model_prefix is not None
        tmp = mx.model.FeedForward.load(model_prefix, args.load_epoch)
        model_args = {'arg_params' : tmp.arg_params,
                      'aux_params' : tmp.aux_params,
                      'begin_epoch' : args.load_epoch}
    # save model
    save_model_prefix = args.save_model_prefix
    if save_model_prefix is None:
        save_model_prefix = model_prefix
    checkpoint = None if save_model_prefix is None else mx.callback.do_checkpoint(save_model_prefix)

    # data
    (train, val) = data_loader(args, kv)

    # train
    devs = mx.cpu() if args.gpus is None else [
        mx.gpu(int(i)) for i in args.gpus.split(',')]

    epoch_size = args.num_examples / args.batch_size

    if args.kv_store == 'dist_sync':
        epoch_size /= kv.num_workers
        model_args['epoch_size'] = epoch_size

    if 'lr_factor' in args and args.lr_factor < 1:
        model_args['lr_scheduler'] = mx.lr_scheduler.FactorScheduler(
            step=max(int(epoch_size * args.lr_factor_epoch), 1),
            factor=args.lr_factor)

    if 'clip_gradient' in args and args.clip_gradient is not None:
        model_args['clip_gradient'] = args.clip_gradient

    # disable kvstore for single device
    if 'local' in kv.type and (
            args.gpus is None or len(args.gpus.split(',')) is 1):
        kv = None

    mod = mx.mod.Module(network, context=devs)

    if eval_metrics is None:
        eval_metrics = ['accuracy']
        # TopKAccuracy only allows top_k > 1
        for top_k in [5, 10, 20]:
            eval_metrics.append(mx.metric.create('top_k_accuracy', top_k=top_k))

    if batch_end_callback is not None:
        if not isinstance(batch_end_callback, list):
            batch_end_callback = [batch_end_callback]
    else:
        batch_end_callback = []
    batch_end_callback.append(mx.callback.Speedometer(args.batch_size, 50))

    mod.fit(train_data=train, eval_metric=eval_metrics, eval_data=val, optimizer='sgd',
            optimizer_params={'learning_rate':args.lr, 'momentum': 0.9, 'wd': 0.00001},
            num_epoch=args.num_epochs, batch_end_callback=batch_end_callback,
            initializer=mx.init.Xavier(factor_type="in", magnitude=2.34),
            kvstore=kv, epoch_end_callback=checkpoint, **model_args)
985645
def tigrprepare(args):
    
    p = OptionParser(tigrprepare.__doc__)
    opts, args = p.parse_args(args)

    if len(args) != 4:
        sys.exit(not p.print_help())

    fastafile, asmbl_id, db, pasa_db = args
    if asmbl_id == 'all':
        idsfile = fastafile + ".ids"
        if need_update(fastafile, idsfile):
            ids([fastafile, "-o", idsfile])
    else:
        idsfile = asmbl_id

    oneid = open(idsfile).next().strip()

    weightsfile = "weights.txt"
    if need_update(idsfile, weightsfile):
        cmd = "$EVM/TIGR-only/create_sample_weights_file.dbi"
        cmd += " {0} {1} | tee weights.txt".format(db, oneid)
        sh(cmd)

    evs = ["gene_predictions.gff3", "transcript_alignments.gff3",
           "protein_alignments.gff3"]
    if need_update(weightsfile, evs):
        cmd = "$EVM/TIGR-only/write_GFF3_files.dbi"
        cmd += " --db {0} --asmbl_id {1} --weights {2}".\
                format(db, idsfile, weightsfile)
        sh(cmd)

    evs[1] = fix_transcript()

    partition(evs)
    runfile = "run.sh"
    contents = EVMRUN.format(*evs)
    write_file(runfile, contents)
523794
def map_predict(interface, state, label, inp):
    
    import numpy as np
    out = interface.output(0)

    continuous = [j for i, j in enumerate(state["X_indices"]) if
                  state["X_meta"][i] == "c"]  # indices of continuous features
    discrete = [j for i, j in enumerate(state["X_indices"]) if
                state["X_meta"][i] == "d"]  # indices of discrete features

    cont = True if len(continuous) > 0 else False  # enables calculation of Gaussian probabilities
    disc = True if len(discrete) > 0 else False  # enables calculation of multinomial probabilities.

    for row in inp:
        row = row.strip().split(state["delimiter"])
        if len(row) > 1:  # if row is empty
            # set id of a sample
            x_id = "" if state["id_index"] == -1 else row[state["id_index"]]
            # initialize prior probability for all labels
            probs = state["fit_model"]["prior_log"]

            if cont:  # continuous features
                x = np.array([(0 if row[j] in state["missing_vals"] else float(row[j])) for j in
                              continuous])  # sets selected features of the sample
                # Gaussian distribution
                probs = probs - 0.5 * np.sum(
                    np.true_divide((x - state["fit_model"]["mean"]) ** 2, state["fit_model"]["var"]) +
                    state["fit_model"]["var_log"], axis=1)

            if disc:  # discrete features
                # multinomial distribution
                probs = probs + np.sum(
                    [(0 if row[i] in state["missing_vals"] else state["fit_model"].get((str(i), row[i]), np.zeros(1)))
                     for i in discrete], axis=0)

            # normalize by P(x) = P(f_1, ..., f_n)
            log_prob_x = np.log(np.sum(np.exp(probs)))
            probs = np.exp(np.array(probs) - log_prob_x)
            # Predicted label is the one with highest probability
            y_predicted = max(zip(probs, state["fit_model"]["y_labels"]))[1]
            out.add(x_id, (y_predicted, probs.tolist()))
251674
def preprocess_example_common(example, mode, hparams):
  """"""
  if "inputs" in example and hparams.max_input_seq_length > 0:
    example["inputs"] = example["inputs"][:hparams.max_input_seq_length]
  if hparams.prepend_mode != "none":
    if mode == tf.estimator.ModeKeys.PREDICT:
      example["partial_targets"] = tf.concat([example["inputs"], [0]], 0)
    else:
      example["targets"] = tf.concat(
          [example["inputs"], [0], example["targets"]], 0)
  if "targets" in example and hparams.max_target_seq_length > 0:
    example["targets"] = example["targets"][:hparams.max_target_seq_length]
  if hparams.split_to_length:
    new_example = {}
    for k, v in six.iteritems(example):
      if k == "targets" or k == "inputs":
        new_example[k] = tf.reshape(v, [-1, hparams.split_to_length, 1, 1])
      else:
        tf.logging.warning("Dropping feature %s" % k)
    return tf.data.Dataset.from_tensor_slices(new_example)
  return example
370941
def _ingest_list(self, input_list, schema_list, path_to_root):

        '''
            
        '''

        valid_list = []

    # construct max list size
        max_size = None
        rules_path_to_root = re.sub('\[\d+\]', '[0]', path_to_root)
        if 'max_size' in self.keyMap[rules_path_to_root].keys():
            if not self.keyMap[rules_path_to_root]['max_size']:
                return valid_list
            else:
                max_size = self.keyMap[rules_path_to_root]['max_size']

    # iterate over items in input list
        if input_list:
            rules_index = self._datatype_classes.index(schema_list[0].__class__)
            rules_type = self._datatype_names[rules_index]
            for i in range(len(input_list)):
                item_path = '%s[%s]' % (path_to_root, i)
                value_match = False
                try:
                    item_index = self._datatype_classes.index(input_list[i].__class__)
                    item_type = self._datatype_names[item_index]
                    if item_type == rules_type:
                        value_match = True
                except:
                    value_match = False
                if value_match:
                    try:
                        if item_type == 'boolean':
                            valid_list.append(self._validate_boolean(input_list[i], item_path))
                        elif item_type == 'number':
                            valid_list.append(self._validate_number(input_list[i], item_path))
                        elif item_type == 'string':
                            valid_list.append(self._validate_string(input_list[i], item_path))
                        elif item_type == 'map':
                            valid_list.append(self._ingest_dict(input_list[i], schema_list[0], item_path))
                        elif item_type == 'list':
                            valid_list.append(self._ingest_list(input_list[i], schema_list[0], item_path))
                    except:
                        pass
                if isinstance(max_size, int):
                    if len(valid_list) == max_size:
                        return valid_list

        return valid_list
631849
def prep_data(data, features, session_id, prediction_window, predictions_in_chunk, target=None, verbose=True):
    
    if target is None:
        target = ""
    if verbose:
        result_dict = _extensions._activity_classifier_prepare_data_verbose(
            data, features, session_id, prediction_window, predictions_in_chunk, target)
    else:
        result_dict = _extensions._activity_classifier_prepare_data(
            data, features, session_id, prediction_window, predictions_in_chunk, target)


    return result_dict["converted_data"] , result_dict["num_of_sessions"]
35106
def prepareToSolve(self):
        '''
        
        '''
        self.setAndUpdateValues(self.solution_next,self.IncomeDstn,self.LivPrb,self.DiscFac)
        self.defBoroCnst(self.BoroCnstArt)