Dataset should be of dimensions (trace, rep, samples)
330194
def dataset_spike_counts(dset, threshold, fs):
    """"""
    if len(dset.shape) == 3:
        results = np.zeros(dset.shape[0])
        for itrace in range(dset.shape[0]):
            results[itrace] = count_spikes(dset[itrace], threshold, fs)
        return results
    elif len(dset.shape == 2):
        return count_spikes(dset, threshold, fs)
    else:
        raise Exception("Improper data dimensions")
585476
def dimensions(filenames):
    """"""

    single = type(filenames) is str
    if single:
        filenames = [filenames]
    dims = get_dimensions(filenames)
    if single:
        dims = dims[0]
    return dims
85562
def with_dimensions(self, *dimensions):
    """"""
    self.dimensions = tuple(
        self._maybe_make_dimension(dim) for dim in dimensions)
    self._cached = None
    return self
825809
def create_dataset(self, group, chunk_size):
        """"""
        group.attrs['format'] = self.dformat
        group.attrs['dim'] = self.dim

        if chunk_size == 'auto':
            group.create_dataset(
                'coordinates', (0, 2), dtype=np.float64,
                chunks=True, maxshape=(None, 2))

            group.create_dataset(
                self.name, (0,), dtype=self.dtype,
                chunks=True, maxshape=(None,))

        else:
            # for storing sparse data we don't use the self.nb_per_chunk,
            # which is only used by the Writer to determine times chunking.
            per_chunk = nb_per_chunk(self.dtype.itemsize, 1, chunk_size)

            group.create_dataset(
                'coordinates', (0, 2), dtype=np.float64,
                chunks=(per_chunk, 2), maxshape=(None, 2))

            group.create_dataset(
                self.name, (0,), dtype=self.dtype,
                chunks=(per_chunk,), maxshape=(None,))

        dtype = np.int64
        if chunk_size == 'auto':
            chunks = True
            self.nb_per_chunk = 'auto'
        else:
            chunks = (nb_per_chunk(np.dtype(dtype).itemsize, 1, chunk_size),)
            # Needed by Times.create_dataset
            self.nb_per_chunk = nb_per_chunk(
                self.dtype.itemsize,
                int(round(self.sparsity*self.dim)),
                chunk_size)

        group.create_dataset(
            'frames', (0,), dtype=dtype,
            chunks=chunks, maxshape=(None,))
124092
def dimensions
      rows = []
      cols = []
      @win.getmaxyx(rows, cols)
      rows = rows.first
      cols = cols.first
      [rows, cols]
    end
1127138
def samples(self, gp, Y_metadata=None):
        
        orig_shape = gp.shape
        gp = gp.flatten()
787683
def ndim(self):
        """"""
        if self.is_raster():
            return {
                FeatureType.DATA: 4,
                FeatureType.MASK: 4,
                FeatureType.SCALAR: 2,
                FeatureType.LABEL: 2,
                FeatureType.DATA_TIMELESS: 3,
                FeatureType.MASK_TIMELESS: 3,
                FeatureType.SCALAR_TIMELESS: 1,
                FeatureType.LABEL_TIMELESS: 1
            }[self]
        return None
591859
def _infer_shape(self, dimensions):
    
    # Size of input
    n = np.prod(dimensions)
    # Size of output where defined
    m = np.prod(abs(np.array(self._shape)))
    # Replace wildcard
    v = np.array(self._shape)
    v[v == -1] = n // m
    return tuple(v)
155809
def _sampleFromEdges(self):
    
    dimensionsToChooseFrom = range(self.dimension)
    random.shuffle(dimensionsToChooseFrom)
    dim1, dim2 = dimensionsToChooseFrom[0], dimensionsToChooseFrom[1]
    coordinates = [random.uniform(-1, 1) * dim / 2. for dim in self.dimensions]
    coordinates[dim1] = self.dimensions[dim1] / 2. * random.choice([-1, 1])
    coordinates[dim2] = self.dimensions[dim2] / 2. * random.choice([-1, 1])
    return coordinates
933808
def _gen_dimension_table(self):
        
        headers = ['Dimension Name', 'Description',
            'Global Size', 'Extents']

        table = []
        for dimval in sorted(self.dimensions(copy=False).itervalues(),
                             key=lambda dval: dval.name.upper()):

            table.append([dimval.name,
                dimval.description,
                dimval.global_size,
                (dimval.lower_extent, dimval.upper_extent)])

        return table, headers
812930
def range(cls, dataset, dimension):
        
        dim = dataset.get_dimension(dimension, strict=True)
        values = dataset.dimension_values(dim.name, False)
        return (np.nanmin(values), np.nanmax(values))
1302365
def dimensions_from_subgroups(s):
    
    letters = 'XYABCDEFGHIJKLMNOPQRSTUVWZ'
    return ' '.join(['dim{0}={1}'.format(dim, sg.name) for dim, sg in zip(letters, s)])
266125
def samples(self):
        """"""
        names = self.series.dimensions
        for values in zip(*(getattr(self.series, name) for name in names)):
            yield dict(zip(names, values))
213192
def fit(self, features, labels):
        """"""

        # setup data
        # imputation
        if self.clean:
            features = self.impute_data(features)
        # save the number of features
        self.n_features = features.shape[1]
        self.n_samples = features.shape[0]
        # set population size
        if type(self.population_size) is str:
            if 'x' in self.population_size: # set pop size prop to features
                self.population_size = int(
                        float(self.population_size[:-1])*features.shape[1])
            else:
                self.population_size = int(self.population_size)

        if self.verbosity >0: print("population size:",self.population_size)
        

        # re-initialize pipeline (needs to be here rather than init for GridSearchCV)
        if self.normalize:
            self.pipeline = Pipeline([('standardScaler',StandardScaler()), ('ml', self.ml)])
        else:
            self.pipeline = Pipeline([('ml',self.ml)])

        # set variable names if they haven't been set
        if self.names is None:
            self.names = ['x_'+str(i) for i in np.arange(features.shape[1])]
        # set variable data types if they haven't been set
        if self.dtypes is None:
            self.dtypes = ['f' for i in np.arange(features.shape[1])]
        
        # create terminal set
        for i in np.arange(self.n_features):
            self.term_set.append(node('x',loc=i,otype=self.dtypes[i])) # features
            # add ephemeral random constants if flag
            if self.erc: # ephemeral random constants
                self.term_set.append(node('k',value=self.random_state.rand()))

        # edit function set if boolean
        if self.boolean or self.otype=='b': # include boolean functions
            self.func_set += [node('!'), node('&'), node('|'), node('=='),
                        node('>_f'), node('<_f'), node('>=_f'), node('<=_f'),
                        node('>_b'), node('<_b'), node('>=_b'), node('<=_b'),
                        node('xor_b'), node('xor_f')]

        # add mdr if specified
        if self.mdr:
            self.func_set += [node('mdr2')]

        # print few settings
        if self.verbosity > 1:
            for arg in self.get_params():
                print('{}\t=\t{}'.format(arg, self.get_params()[arg]))
            print('')
        
        ######################################################### initial model
        # fit to original data
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            if self.scoring_function == roc_auc_score:
                self._best_score = self.roc_auc_cv(features,labels)
            else:
                self._best_score = np.mean(
                                   [self.scoring_function(labels[test],
                                       self.pipeline.fit(features[train],labels[train]).
                                                     predict(features[test]))
                                   for train, test in KFold().split(features,
                                                                     labels)])

        initial_score = self._best_score
        if self.verbosity > 0:
            print("initial ML CV: {:1.3f}".format(self._best_score))

        
        ############################################# Create initial population
        # for now, force seed_with_ml to be off if otype is 'b', since data
        # types are assumed to be float
        if self.otype=='b':
            self.seed_with_ml = False
        self.pop = self.init_pop()
        # check that uuids are unique in population
        uuids = [p.id for p in self.pop.individuals]
        if len(uuids) != len(set(uuids)):
            pdb.set_trace()
        # Evaluate the entire population
        # X represents a matrix of the population outputs (number of samples x
        # population size)
        # single thread
        self.X = self.transform(features,self.pop.individuals,labels).transpose()
        # pdb.set_trace()
        # parallel:
        # X = np.asarray(Parallel(n_jobs=-1)(
        # delayed(out)(I,features,self.otype,labels) for I in self.pop.individuals),
        # order = 'F')

        # calculate fitness of individuals
        # fitnesses = list(map(lambda I: fitness(I,labels,self.pipeline),X))
        self.F = self.calc_fitness(self.X,labels,self.fit_choice,self.sel)
        #pdb.set_trace()
        #with Parallel(n_jobs=10) as parallel:
        ####################

        self.diversity=[]
        # progress bar
        pbar = tqdm(total=self.generations,disable = self.verbosity==0,
                    desc='Internal CV: {:1.3f}'.format(self._best_score))
        stall_count = 0
        ########################################################### main GP loop
        for g in np.arange(self.generations):
            if stall_count == self.max_stall:
                if self.verbosity > 0: print('max stall count reached.')
                break

            if self.track_diversity:
                self.get_diversity(self.X)

            # mid verbosity printouts
            if self.verbosity > 1:
                
                print("generation", str(g))          
            
                print("median fitness pop: %0.2f" % np.median(
                        [np.mean(f) for f in self.F]))
            
                print("best fitness pop: %0.2f" % np.min(
                    [np.mean(f) for f in self.F]))
                if self.track_diversity:
                    print("feature diversity: %0.2f" % self.diversity[-1])

            # high verbosity printouts    
            if self.verbosity > 2:
                eqns = self.stacks_2_eqns(self.pop.individuals)
                fs = [np.mean(f) for f in self.F] 
                print("population:",[("%0.2f" % f, eqns[i]) for f,i in 
                                     zip(np.sort(fs), np.argsort(fs))])
                #print("pop fitnesses:", ["%0.2f" % np.mean(f) for f in self.F])
            
            ####################################################### fit ml model
            if self.verbosity > 1: print("ml fitting...")

            tmp_score=0
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                try:
                    if self.valid_loc():
                        if self.scoring_function == roc_auc_score:
                            tmp_score =  self.roc_auc_cv(self.X[self.valid_loc(),:].transpose(),
                                                         labels)
                        else:
                            tmp_score =  np.mean(
                                [self.scoring_function(labels[test], self.pipeline.fit(
                                self.X[self.valid_loc(),:].transpose()[train], labels[train]).
                                predict(self.X[self.valid_loc(),:].transpose()[test]))
                                    for train, test in KFold().split(features,
                                                                     labels)])

                except ValueError as detail:
                    print("warning: ValueError in ml fit. X.shape:",
                          self.X[:,self.valid_loc()].transpose().shape,
                          "labels shape:",labels.shape)
                    print("First ten entries X:",
                          self.X[self.valid_loc(),:].transpose()[:10])
                    print("First ten entries labels:",labels[:10])
                    print("equations:",self.stacks_2_eqns(self.pop.individuals))
                    print("FEW parameters:",self.get_params())
                    print("---\ndetailed error message:",
                                                 detail)
                    raise(detail)
                
            if self.verbosity > 1:
                print("current ml validation score:",tmp_score)

            #################################################### save best model
            if self.valid_loc() and tmp_score > self._best_score:
                self._best_estimator = copy.deepcopy(self.pipeline)
                self._best_score = tmp_score
                stall_count = 0;
                self._best_inds = copy.deepcopy(self.valid())
                if self.verbosity > 1:
                    print("updated best internal CV:",self._best_score)
            else:
                stall_count = stall_count + 1

            ########################################################## variation
            if self.verbosity > 2:
                print("variation...")
            offspring,elite,elite_index = self.variation(self.pop.individuals)

            ################################################# evaluate offspring
            if self.verbosity > 2:
                print("output...")
            X_offspring = self.transform(features,offspring).transpose()

            if self.verbosity > 2:
                print("fitness...")
            F_offspring = self.calc_fitness(X_offspring,
                                            labels,self.fit_choice,self.sel)

            ########################################################### survival
            if self.verbosity > 2: print("survival..")
            survivors,survivor_index = self.survival(self.pop.individuals, offspring, 
                                                     elite, elite_index,
                                                     X = self.X, X_O=X_offspring,
                                                     F=self.F, F_O=F_offspring)
            # set survivors
            self.pop.individuals[:] = survivors
            self.X = np.vstack((self.X, X_offspring))[survivor_index]
            if 'lexicase' in self.sel:
                self.F = np.asarray(
                        np.vstack((self.F, F_offspring))[survivor_index],
                        order='F')
            else:
                self.F = np.asarray(
                        np.hstack((self.F,F_offspring))[survivor_index],
                        order='F')

            if self.verbosity > 2:
                print("median fitness survivors: %0.2f" % np.median(
                        [np.mean(f) for f in self.F]))
            if self.verbosity>2:
                print("best features:",
                      self.stacks_2_eqns(self._best_inds) if self._best_inds
                      else 'original')
            pbar.set_description('Internal CV: {:1.3f}'.format(self._best_score))
            pbar.update(1)
        # end of main GP loop
            ####################
        if self.verbosity > 0: print('finished. best internal val score:'
                                     ' {:1.3f}'.format(self._best_score))
        if self.verbosity > 0: print("final model:\n",self.print_model())

        if not self._best_estimator:
            # if no better model found, just return underlying method fit to the
            # training data
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                self._best_estimator = self.pipeline.fit(features,labels)
        else:
            # fit final estimator to all the training data
            with warnings.catch_warnings():
                warnings.simplefilter("ignore")
                self._best_estimator.fit(self.transform(features),labels)
        return self
1248681
def _correct_dims(data):
        """"""
        if not hasattr(data, 'dims'):
            raise TypeError("Data must have a 'dims' attribute.")

        # doesn't actually copy the data underneath
        # we don't want our operations to change the user's data
        data = data.copy()

        if 'y' not in data.dims or 'x' not in data.dims:
            if data.ndim != 2:
                raise ValueError("Data must have a 'y' and 'x' dimension")

            # rename dimensions so we can use them
            # don't rename 'x' or 'y' if they already exist
            if 'y' not in data.dims:
                # find a dimension that isn't 'x'
                old_dim = [d for d in data.dims if d != 'x'][0]
                data = data.rename({old_dim: 'y'})
            if 'x' not in data.dims:
                # find a dimension that isn't 'y'
                old_dim = [d for d in data.dims if d != 'y'][0]
                data = data.rename({old_dim: 'x'})

        if "bands" not in data.dims:
            if data.ndim <= 2:
                data = data.expand_dims('bands')
                data['bands'] = ['L']
            else:
                raise ValueError("No 'bands' dimension provided.")

        return data
213983
def load_dataset(data_name):
    """"""
    if data_name == 'MR' or data_name == 'Subj':
        train_dataset, output_size = _load_file(data_name)
        vocab, max_len = _build_vocab(data_name, train_dataset, [])
        train_dataset, train_data_lengths = _preprocess_dataset(train_dataset, vocab, max_len)
        return vocab, max_len, output_size, train_dataset, train_data_lengths
    else:
        train_dataset, test_dataset, output_size = _load_file(data_name)
        vocab, max_len = _build_vocab(data_name, train_dataset, test_dataset)
        train_dataset, train_data_lengths = _preprocess_dataset(train_dataset, vocab, max_len)
        test_dataset, test_data_lengths = _preprocess_dataset(test_dataset, vocab, max_len)
        return vocab, max_len, output_size, train_dataset, train_data_lengths, test_dataset, \
               test_data_lengths
470664
def sample_given_context(self, c, c_dims):
        '''
        
        '''
        index = self.discrete_progress.sample_given_context(c, c_dims, self.space)
        return self.space.rand_value(index).flatten()[list(set(range(len(self.space.cardinalities))) - set(c_dims))]