Perform a Students t-test on the two sets of revision data.

    See the analyze() function for a description of the `weight_fn` argument.
967553
def calc_t(w1, w2, weight_fn=None):
    
    if not w1 or not w2:
        return 0

    s1 = analyze(w1, weight_fn)
    s2 = analyze(w2, weight_fn)
    delta_s = s2['avg'] - s1['avg']

    if delta_s == 0:
        return 0
    if s1['variance'] == 0 and s2['variance'] == 0:
        return float('inf')

    return delta_s / (((s1['variance'] / s1['n']) + (s2['variance'] / s2['n'])) ** 0.5)
399471
def _capture(f, t, t0, factor):
    '''
    
    '''
    n_per_sec = len(t) / t[-1]

    # len of one split:
    n = int(t0 * factor * n_per_sec)
    s = len(f) // n
    m = s * n
    f = f[:m]
    ff = np.split(f, s)
    m = np.mean(ff, axis=1)

    return np.std(m)
1003763
def validate_examples(example_file):
    

    def test_example(raw):
        example = tf.train.Example()
        example.ParseFromString(raw)

        pi = np.frombuffer(example.features.feature['pi'].bytes_list.value[0], np.float32)
        value = example.features.feature['outcome'].float_list.value[0]
        assert abs(pi.sum() - 1) < 1e-4, pi.sum()
        assert value in (-1, 1), value

    opts = tf.python_io.TFRecordOptions(tf.python_io.TFRecordCompressionType.ZLIB)
    for record in tqdm(tf.python_io.tf_record_iterator(example_file, opts)):
        test_example(record)
1199062
def _analyze(self):
        
        if self.value is None or self.value == self.previous:
            pass

        elif self._operation == "append":
            self._append = self.value

        elif self._operation == "prepend":
            self._prepend = self.value

        elif self.previous is None:
            self._assignments = self.value

        elif len(self.value) < len(self.previous):
            # if elements have been removed,
            # rewrite the whole list
            self._assignments = self.value

        elif len(self.previous) == 0:
            # if we're updating from an empty
            # list, do a complete insert
            self._assignments = self.value
        else:

            # the max start idx we want to compare
            search_space = len(self.value) - max(0, len(self.previous) - 1)

            # the size of the sub lists we want to look at
            search_size = len(self.previous)

            for i in range(search_space):
                # slice boundary
                j = i + search_size
                sub = self.value[i:j]
                idx_cmp = lambda idx: self.previous[idx] == sub[idx]
                if idx_cmp(0) and idx_cmp(-1) and self.previous == sub:
                    self._prepend = self.value[:i] or None
                    self._append = self.value[j:] or None
                    break

            # if both append and prepend are still None after looking
            # at both lists, an insert statement will be created
            if self._prepend is self._append is None:
                self._assignments = self.value

        self._analyzed = True
1121676
def fit(self, train_set, test_set):
        
        with tf.Graph().as_default(), tf.Session() as self.tf_session:
            self.build_model()
            tf.global_variables_initializer().run()
            third = self.num_epochs // 3

            for i in range(self.num_epochs):
                lr_decay = self.lr_decay ** max(i - third, 0.0)
                self.tf_session.run(
                    tf.assign(self.lr_var, tf.multiply(self.learning_rate, lr_decay)))

                train_perplexity = self._run_train_step(train_set, 'train')
                print("Epoch: %d Train Perplexity: %.3f"
                      % (i + 1, train_perplexity))

            test_perplexity = self._run_train_step(test_set, 'test')
            print("Test Perplexity: %.3f" % test_perplexity)
1373201
def __score_method(X, y, fcounts, model_generator, score_function, method_name, nreps=10, test_size=100, cache_dir="/tmp"):
    

    old_seed = np.random.seed()
    np.random.seed(3293)

    # average the method scores over several train/test splits
    method_reps = []

    data_hash = hashlib.sha256(__toarray(X).flatten()).hexdigest() + hashlib.sha256(__toarray(y)).hexdigest()
    for i in range(nreps):
        X_train, X_test, y_train, y_test = train_test_split(__toarray(X), y, test_size=test_size, random_state=i)

        # define the model we are going to explain, caching so we onlu build it once
        model_id = "model_cache__v" + "__".join([__version__, data_hash, model_generator.__name__])+".pickle"
        cache_file = os.path.join(cache_dir, model_id + ".pickle")
        if os.path.isfile(cache_file):
            with open(cache_file, "rb") as f:
                model = pickle.load(f)
        else:
            model = model_generator()
            model.fit(X_train, y_train)
            with open(cache_file, "wb") as f:
                pickle.dump(model, f)

        attr_key = "_".join([model_generator.__name__, method_name, str(test_size), str(nreps), str(i), data_hash])
        def score(attr_function):
            def cached_attr_function(X_inner):
                if attr_key not in _attribution_cache:
                    _attribution_cache[attr_key] = attr_function(X_inner)
                return _attribution_cache[attr_key]

            #cached_attr_function = lambda X: __check_cache(attr_function, X)
            if fcounts is None:
                return score_function(X_train, X_test, y_train, y_test, cached_attr_function, model, i)
            else:
                scores = []
                for f in fcounts:
                    scores.append(score_function(f, X_train, X_test, y_train, y_test, cached_attr_function, model, i))
                return np.array(scores)

        # evaluate the method (only building the attribution function if we need to)
        if attr_key not in _attribution_cache:
            method_reps.append(score(getattr(methods, method_name)(model, X_train)))
        else:
            method_reps.append(score(None))

    np.random.seed(old_seed)
    return np.array(method_reps).mean(0)
1200876
def with_sample_weight(clf, sample_weight, fit_params):
    
    param_name = _get_classifier_prefix(clf) + "sample_weight"
    params = {param_name: sample_weight}
    params.update(fit_params)
    return params
422271
def _poor_convergence(z, r, f, bn, mvec):
    
    check_points = (-0.4 + 0.3j, 0.7 + 0.2j, 0.02 - 0.06j)
    diffs = []
    ftests = []
    for check_point in check_points:
        rtest = r * check_point
        ztest = z + rtest
        ftest = f(ztest)
        # Evaluate powerseries:
        comp = np.sum(bn * np.power(check_point, mvec))
        ftests.append(ftest)
        diffs.append(comp - ftest)

    max_abs_error = np.max(np.abs(diffs))
    max_f_value = np.max(np.abs(ftests))
    return max_abs_error > 1e-3 * max_f_value
1043978
def reeval(self, X, fit, func, ask, args=()):
        
        self.fit = list(fit)
        self.fitre = list(fit)
        self.idx = self.indices(fit)
        if not len(self.idx):
            return self.idx
        evals = int(self.evaluations) if self.f_aggregate else 1
        fagg = np.median if self.f_aggregate is None else self.f_aggregate
        for i in self.idx:
            X_i = X[i]
            if self.epsilon:
                if self.parallel:
                    self.fitre[i] = fagg(func(ask(evals, X_i, self.epsilon), *args))
                else:
                    self.fitre[i] = fagg([func(ask(1, X_i, self.epsilon)[0], *args)
                                            for _k in xrange(evals)])
            else:
                self.fitre[i] = fagg([func(X_i, *args) for _k in xrange(evals)])
        self.evaluations_just_done = evals * len(self.idx)
        return self.fit, self.fitre, self.idx
933951
def _check_list_weights(parameter, name):
    '''
    
    '''
    if not isinstance(parameter, list):
        raise ValueError('%s must be formatted with a list of tuples' % name)
    weight = np.sum([val[1] for val in parameter])
    if fabs(weight - 1.) > 1E-8:
        raise ValueError('%s weights do not sum to 1.0!' % name)
    return parameter
793664
def compare(self, other, analysis_set, output_directory, unique_ajps, verbose = True, compile_pdf = True):
        
        self_unique_name = self.get_definitive_name(unique_ajps)
        other_unique_name = other.get_definitive_name(unique_ajps)

        output_directory = os.path.join(output_directory, os.path.join(self_unique_name, other_unique_name) )
        assert( not os.path.isdir( output_directory ) )

        subplot_directory = os.path.join(output_directory, 'plots')
        if not os.path.isdir(subplot_directory):
            os.makedirs(subplot_directory)

        report = lr.LatexReport( table_of_contents = False )

        # Construct dataframe comparing our predictions to other's predictions
        both_predictions = pandas.concat(
            [
                self.add_identifying_columns_to_df(self.dataframe[['Predicted']], unique_ajps),
                other.add_identifying_columns_to_df(other.dataframe[['Predicted']], unique_ajps),
            ],
            join = 'outer',
        ).sort_index()
        both_predictions_subtracted = subtract_row_pairs_for_display(
            both_predictions,
            output_csv = os.path.join(output_directory, 'predictions_v_predictions.csv'),
            merge_df = self.dataframe,
            verbose = verbose,
        )

        # Construct dataframe comparing our diff with experimental values to other's
        self_diff = self.get_pred_minus_exp_dataframe(analysis_set, unique_ajps)
        other_diff = other.get_pred_minus_exp_dataframe(analysis_set, unique_ajps)
        diffs_df = pandas.concat(
            [self_diff, other_diff],
            join = 'outer',
        )

        diffs_df = subtract_row_pairs_for_display(
            diffs_df,
            output_csv = os.path.join(output_directory, 'diffs_v_diffs.csv'),
            merge_df = self.dataframe,
            verbose = verbose,
        )

        report.set_title_page(
            '%s vs %s' % (
                self.get_definitive_name(unique_ajps, join_character = '\n'),
                other.get_definitive_name(unique_ajps, join_character = '\n')
            )
        )
        predictions_v_predictions_df = self.dataframe[['Predicted']].merge(
            other.dataframe[['Predicted']],
            left_index = True,
            right_index = True,
        )
        predictions_v_predictions_df.columns = [self_unique_name, other_unique_name]
        report.add_plot( general_matplotlib.make_corr_plot(predictions_v_predictions_df, predictions_v_predictions_df.columns.values[0], predictions_v_predictions_df.columns.values[1], output_directory = subplot_directory, plot_title = 'Prediction comparison', axis_label_size = 8.0, output_name = 'vs_scatter', fig_height = 7, fig_width = 8, verbose = verbose, plot_11_line = True ), plot_title = 'Experimental vs. Predicted scatterplot (with density binning)' )

        diff_v_diff_dataframe = self.get_pred_minus_exp_dataframe(analysis_set).merge(
            other.get_pred_minus_exp_dataframe(analysis_set),
            left_index = True,
            right_index = True,
        )
        report.add_section_page( title = 'Plots' )
        diff_v_diff_dataframe.columns = [self_unique_name, other_unique_name]
        report.add_plot( general_matplotlib.make_corr_plot(diff_v_diff_dataframe, diff_v_diff_dataframe.columns.values[0], diff_v_diff_dataframe.columns.values[1], output_directory = subplot_directory, plot_title = 'Error v. Error', axis_label_size = 7.0, output_name = 'diff_vs_scatter', fig_height = 7, fig_width = 8, verbose = verbose, plot_11_line = True ), plot_title = 'Outliers --- Error (Predicted - Experimental) v. error. \\ x-axis=%s \\ y-axis=%s' % (diff_v_diff_dataframe.columns.values[0], diff_v_diff_dataframe.columns.values[1]) )

        report.add_section_page( title = 'Tables' )

        report.content.append( lr.LatexPandasTable(
            diffs_df, float_format = float_format_2sigfig,
            caption_text = 'Outliers --- Comparison of error (Predicted - Experimental) for first prediction set (%s) vs second set of predictions (%s). Values sorted by descending absolute delta.' % (self_unique_name, other_unique_name),
        ) )

        report.content.append( lr.LatexPandasTable(
            both_predictions_subtracted, float_format = float_format_2sigfig,
            caption_text = 'Direct comparison of predicted values. Values sorted by descending absolute delta.',
        ) )

        # Get joined stats comparison dataframe
        for case_table in BenchmarkRun.make_case_description_tables(BenchmarkRun.get_stats_comparison_dataframe(
                [self, other], unique_ajps,
                output_csv = os.path.join(output_directory, 'comparison_metrics.csv'),
        )):
            report.content.append( case_table )

        report.generate_pdf_report(
            os.path.join(output_directory, 'comparison.pdf'),
            verbose = verbose,
            compile_pdf = compile_pdf,
        )
        if verbose:
            print 'Comparison report saved to:', os.path.join(output_directory, 'comparison.pdf')
        return report
635294
def analyze_theory(V, x0list=[], plot=False):
    
    # initialize path integral
    T = 4.
    ndT = 8.         # use larger ndT to reduce discretization error (goes like 1/ndT**2)
    neval = 3e5   # should probably use more evaluations (10x?)
    nitn = 6
    alpha = 0.1     # damp adaptation

    # create integrator and train it (no x0list)
    integrand = PathIntegrand(V=V, T=T, ndT=ndT)
    integ = vegas.Integrator(integrand.region, alpha=alpha)
    integ(integrand, neval=neval, nitn=nitn / 2, alpha=2 * alpha)

    # evaluate path integral with trained integrator and x0list
    integrand = PathIntegrand(V=V, x0list=x0list, T=T, ndT=ndT)
    results = integ(integrand, neval=neval, nitn=nitn, alpha=alpha)
    print(results.summary())
    E0 = -np.log(results['exp(-E0*T)']) / T
    print('Ground-state energy = %s    Q = %.2f\n' % (E0, results.Q))

    if len(x0list) <= 0:
        return E0
    psi2 = results['exp(-E0*T) * psi(x0)**2'] / results['exp(-E0*T)']
    print('%5s  %-12s %-10s' % ('x', 'psi**2', 'sho-exact'))
    print(27 * '-')
    for i, (x0i, psi2i) in enumerate(zip(x0list, psi2)):
        exact = np.exp(- x0i ** 2) / np.sqrt(np.pi) #* np.exp(-T / 2.)
        print(
            "%5.1f  %-12s %-10.5f"
            % (x0i, psi2i, exact)
            )
    if plot:
        plot_results(E0, x0list, psi2, T)
    return E0
213742
def create_tpu_eval_metrics_fn(problem, model_hparams):
  """"""

  metric_fns = []
  eval_metrics = problem.eval_metric_fns(model_hparams)

  tm = _create_target_modality(problem.get_hparams(model_hparams).modality)
  if isinstance(tm, dict):
    for k, v in six.iteritems(tm):
      weights_fn = modalities.get_weights_fn(v)

      def make_metric_fn(metric_fn):
        def wrapped_metric_fn(logits, labels, features, weights_fn=weights_fn):
          kwargs = {}
          args, _, keywords, _ = inspect.getargspec(metric_fn)
          if ("features" in args) or keywords:
            kwargs["features"] = features
          num, den = metric_fn(logits, labels, weights_fn=weights_fn, **kwargs)
          return tf.metrics.mean(num, den)

        return wrapped_metric_fn

      for metric, metric_fn in six.iteritems(eval_metrics):
        if metric in TPU_METRIC_BLACKLIST:
          log_warn("Skipping eval metric %s in TPU_METRIC_BLACKLIST", metric)
          continue
        name = "%s/metrics-%s/%s" % (k, problem.name, metric)
        metric_fns.append((name, make_metric_fn(metric_fn)))
  else:
    weights_fn = modalities.get_weights_fn(tm)

    def make_metric_fn(metric_fn):
      def wrapped_metric_fn(logits, labels, features):
        kwargs = {}
        args, _, keywords, _ = inspect.getargspec(metric_fn)
        if ("features" in args) or keywords:
          kwargs["features"] = features
        num, den = metric_fn(logits, labels, weights_fn=weights_fn, **kwargs)
        return tf.metrics.mean(num, den)

      return wrapped_metric_fn

    for metric, metric_fn in six.iteritems(eval_metrics):
      if metric in TPU_METRIC_BLACKLIST:
        log_warn("Skipping eval metric %s in TPU_METRIC_BLACKLIST", metric)
        continue
      name = "metrics-%s/%s" % (problem.name, metric)
      metric_fns.append((name, make_metric_fn(metric_fn)))

  def all_metrics_fn(**kwargs):
    """Construct metrics dictionary."""

    original_kwargs = _unflatten_dict(kwargs, prefixes=["logits", "features"])
    del kwargs

    logits = original_kwargs["logits"]
    labels = original_kwargs["labels"]
    features = original_kwargs["features"]
    del original_kwargs

    metrics_dict = {}

    for name, fn in metric_fns:
      if isinstance(logits, dict) and isinstance(labels, dict):
        for k, v in six.iteritems(logits):
          metrics_dict["%s/%s" % (k, name)] = fn(v, labels[k], features)
      elif isinstance(logits, dict):
        tf.logging.warning("Logits is a dict, but labels is not; only "
                           "evaluating logits['targets'] against labels.")
        metrics_dict["%s/%s" % ("targets", name)] = fn(logits["targets"],
                                                       labels, features)
      else:
        metrics_dict[name] = fn(logits, labels, features)

    return metrics_dict

  return all_metrics_fn
800062
def check_weights(nodes_with_a_weight):
    
    weights = [n['weight'] for n in nodes_with_a_weight]
    if abs(sum(weights) - 1.) > PRECISION:
        raise ValueError('The weights do not sum up to 1: %s' % weights)
    return nodes_with_a_weight
91461
def paired(x, y):
    
    samples = ''
    while samples not in ['i', 'r', 'I', 'R', 'c', 'C']:
        print('\nIndependent or related samples, or correlation (i,r,c): ',)
        samples = raw_input()

    if samples in ['i', 'I', 'r', 'R']:
        print('\nComparing variances ...',)
        # USE O'BRIEN'S TEST FOR HOMOGENEITY OF VARIANCE, Maxwell & delaney, p.112
        r = obrientransform(x, y)
        f, p = F_oneway(pstat.colex(r, 0), pstat.colex(r, 1))
        if p < 0.05:
            vartype = 'unequal, p=' + str(round(p, 4))
        else:
            vartype = 'equal'
        print(vartype)
        if samples in ['i', 'I']:
            if vartype[0] == 'e':
                t, p = ttest_ind(x, y, 0)
                print('\nIndependent samples t-test:  ', round(t, 4), round(p, 4))
            else:
                if len(x) > 20 or len(y) > 20:
                    z, p = ranksums(x, y)
                    print('\nRank Sums test (NONparametric, n>20):  ', round(z, 4), round(p, 4))
                else:
                    u, p = mannwhitneyu(x, y)
                    print('\nMann-Whitney U-test (NONparametric, ns<20):  ', round(u, 4), round(p, 4))
        else:  # RELATED SAMPLES
            if vartype[0] == 'e':
                t, p = ttest_rel(x, y, 0)
                print('\nRelated samples t-test:  ', round(t, 4), round(p, 4))
            else:
                t, p = ranksums(x, y)
                print('\nWilcoxon T-test (NONparametric):  ', round(t, 4), round(p, 4))
    else:  # CORRELATION ANALYSIS
        corrtype = ''
        while corrtype not in ['c', 'C', 'r', 'R', 'd', 'D']:
            print('\nIs the data Continuous, Ranked, or Dichotomous (c,r,d): ',)
            corrtype = raw_input()
        if corrtype in ['c', 'C']:
            m, b, r, p, see = linregress(x, y)
            print('\nLinear regression for continuous variables ...')
            lol = [['Slope', 'Intercept', 'r', 'Prob', 'SEestimate'], [round(m, 4), round(b, 4), round(r, 4), round(p, 4), round(see, 4)]]
            pstat.printcc(lol)
        elif corrtype in ['r', 'R']:
            r, p = spearmanr(x, y)
            print('\nCorrelation for ranked variables ...')
            print("Spearman's r: ", round(r, 4), round(p, 4))
        else: # DICHOTOMOUS
            r, p = pointbiserialr(x, y)
            print('\nAssuming x contains a dichotomous variable ...')
            print('Point Biserial r: ', round(r, 4), round(p, 4))
    print('\n\n')
    return None
1034645
def main(argv=None):
    
    import getopt

    # default values
    test_read = None
    test_write = None
    n = 3       # number of repeat

    if argv is None:
        argv = sys.argv[1:]

    try:
        opts, args = getopt.getopt(argv, 'hn:rw', [])

        for o,a in opts:
            if o == '-h':
                print(HELP)
                return 0
            elif o == '-n':
                n = int(a)
            elif o == '-r':
                test_read = True
            elif o == '-w':
                test_write = True

        if args:
            raise getopt.GetoptError('Arguments are not used.')
    except getopt.GetoptError:
        msg = sys.exc_info()[1]     # current exception
        txt = str(msg)
        print(txt)
        return 1

    if (test_read, test_write) == (None, None):
        test_read = test_write = True

    m = Measure(n, test_read, test_write)
    m.measure_all()
    m.print_report()

    return 0
1330731
def _evaluate_bg(self, n, xdata, p=None):
        
        # by default, use the fit values, otherwise, use the guess values.
        if   p is None and self.results is not None: p = self.results[0]
        elif p is None and self.results is None:     p = self._pguess

        # return None if there is no background function
        if self.bg[n] is None: return None

        # assemble the arguments for the function
        args = (xdata,) + tuple(p)

        # evaluate the function
        return self.bg[n](*args)