Store entities and their attributes

        Args:
            df (pandas.DataFrame): data to store (storing appends 'id' and 'type' columns!)
            attribute_columns (list(str)): list of column labels that define attributes
335437
def store(self, df, attribute_columns):
        

        # ID start values depend on currently stored entities/attributes!
        entity_id_start = models.Entity.get_max_id(self.session) + 1
        attribute_id_start = models.Attribute.get_max_id(self.session) + 1

        # append ID and type columns
        df['id'] = range(entity_id_start, entity_id_start + len(df))
        df['type'] = self.type

        # store entities
        df[['id', 'type']].to_sql(name=models.Entity.__tablename__,
                                  con=self.client.engine,
                                  if_exists='append',
                                  index=False)

        # store attributes
        for col in attribute_columns:
            # ID column of df is the entity ID of the attribute
            attr_df = df[[col, 'id']].rename(columns={'id': 'entity_id',
                                                      col: 'value'})
            attr_df['name'] = col

            # add entity ID column, need to respect already existing entities
            attr_df['id'] = range(attribute_id_start, attribute_id_start + len(df))
            attribute_id_start += len(df)

            # store
            attr_df.to_sql(name=models.Attribute.__tablename__,
                           con=self.client.engine,
                           if_exists='append',
                           index=False)
1360092
def types_(self, col: str) -> pd.DataFrame:
        
        cols = self.df.columns.values
        all_types = {}
        for col in cols:
            local_types = []
            for i, val in self.df[col].iteritems():
                #print(i, val, type(val))
                t = type(val).__name__
                if t not in local_types:
                    local_types.append(t)
            all_types[col] = (local_types, i)
        df = pd.DataFrame(all_types, index=["type", "num"])
        return df
1141322
def from_df(cls, df):
        """"""
        t = cls()
        labels = df.columns
        for label in df.columns:
            t.append_column(label, df[label])
        return t
1089224
def update(self,**kwargs):
		
		if 'columns' in kwargs:
			self._d=ta._ohlc_dict(self.df,columns=kwargs.pop('columns',None))
		schema=self._get_schema()
		annotations=kwargs.pop('annotations',None)

		if annotations:
			self.layout['annotations']['values']=utils.make_list(annotations)
		for k,v in list(kwargs.items()):
			try:
				utils.dict_update(self.__dict__,k,v,schema)
			except:
				self.kwargs.update({k:v})
549561
def save_dataframes(self, outdir, prefix='df_'):
        
        # Get list of attributes that start with "df_"
        dfs = list(filter(lambda x: x.startswith(prefix), dir(self)))

        counter = 0
        for df in dfs:
            outpath = ssbio.utils.outfile_maker(inname=df, outext='.csv', outdir=outdir)
            my_df = getattr(self, df)
            if not isinstance(my_df, pd.DataFrame):
                raise TypeError('{}: object is not a Pandas DataFrame'.format(df))

            if my_df.empty:
                log.debug('{}: empty dataframe, not saving'.format(df))
            else:
                my_df.to_csv(outpath)
                log.debug('{}: saved dataframe'.format(outpath))
                counter += 1

        log.debug('Saved {} dataframes at {}'.format(counter, outdir))
825012
def extend_columns(self, column_data, kind):
        
        # 'column_data' is a generator object. It can throw an exception while
        # being consumed. This could happen if the user has launched the app
        # without specifying a database name. This exception must be handled to
        # prevent crashing.
        try:
            column_data = [self.escaped_names(d, '"') for d in column_data]
        except Exception:
            column_data = []

        metadata = self.dbmetadata[kind]
        for relname, column in column_data:
            metadata[self.dbname][relname].append(column)
            self.all_completions.add(column)
304709
def get_custom_label_attribute(self, attribute):
        

        if attribute not in ann_label_fields:
            raise ValueError('Invalid attribute specified')

        if isinstance(self.custom_labels, pd.DataFrame):
            if 'label_store' not in list(self.custom_labels):
                raise ValueError('label_store not defined in custom_labels')
            a = list(self.custom_labels[attribute].values)
        else:
            if len(self.custom_labels[0]) == 2:
                if attribute == 'label_store':
                    raise ValueError('label_store not defined in custom_labels')
                elif attribute == 'symbol':
                    a = [l[0] for l in self.custom_labels]
                elif attribute == 'description':
                    a = [l[1] for l in self.custom_labels]
            else:
                if attribute == 'label_store':
                    a = [l[0] for l in self.custom_labels]
                elif attribute == 'symbol':
                    a = [l[1] for l in self.custom_labels]
                elif attribute == 'description':
                    a = [l[2] for l in self.custom_labels]

        return a
767638
def ColumnAttr(name='', attr=None, **kwargs):
    ''
    return Column(name,
                  expr=attr if attr is not None else name,
                  getter=lambda col,row: getattrdeep(row, col.expr),
                  setter=lambda col,row,val: setattrdeep(row, col.expr, val),
                  **kwargs)
1310136
def df(objs, labels=None):
    
    import pandas as pd
    from .objects import Object, DynamicObject
    if objs:
        objs = list(objs)
        obj = objs[0]
        if isinstance(obj, Object):
            df = pd.DataFrame.from_records(o.tuple() for o in objs)
            df.columns = obj.__class__.defaults
        elif isinstance(obj, DynamicObject):
            df = pd.DataFrame.from_records(o.__dict__ for o in objs)
        else:
            df = pd.DataFrame.from_records(objs)
        if isinstance(obj, tuple) and hasattr(obj, '_fields'):
            # assume it's a namedtuple
            df.columns = obj.__class__._fields
    else:
        df = None
    if labels:
        exclude = [label for label in df if label not in labels]
        df = df.drop(exclude, axis=1)
    return df
1133162
def save_to_store(self):
        
        if not self._store:
            raise AttributeError('No datastore defined!')
        saved_data = self.save_to_data(in_place=True)
        data = Serializer.serialize(saved_data)
        self._store.store_blob(data, 'all_keys_with_undefined')
911019
def _attribute_definition(self, attrid):
        
        attrs = self._schema["attributes"]

        try:
            # Make a new dict to avoid side effects
            return dict(attrs[attrid])
        except KeyError:
            attr_names = self._schema["attribute_names"]
            attrdef = attrs.get(attr_names.get(str(attrid).lower()))

            if not attrdef:
                return None
            else:
                return dict(attrdef)
902964
def get_fields_frame(schema):
    '''
    
    '''
    fields = []

    def get_field_record(i, key, value, parents):
        if value.get('private') or 'default' not in value:
            raise Skip
        field = (len(parents) - 1, i, tuple(parents[:-1]), parents[-1],
                 value['type'], value['default'])
        return fields.append(field)

    # Fill fields list.
    get_types(schema['properties'], get_field_record)

    # Order by level, then explicit index.
    df_fields = pd.DataFrame(sorted(fields), columns=['level_i', 'field_i',
                                                      'parents', 'field',
                                                      'field_type', 'default'])

    df_fields['attributes'] = df_fields.apply(lambda row:
                                              get_nested_item(row, schema
                                                              ['properties'],
                                                              row.parents +
                                                              (row.field, )),
                                              axis=1)
    return df_fields
1051850
def _get_str_columns(sf):
    
    return [name for name in sf.column_names() if sf[name].dtype == str]
467532
def _construct_columns(self, column_map):
        
        '''  '''
        
        from sqlalchemy import Column, String, Boolean, Integer, Float, Binary 
    
        column_args = []
        for key, value in column_map.items():
            record_key = value[0]
            datatype = value[1]
            max_length = value[2]
            if record_key == 'id':
                if datatype in ('string', 'float', 'integer'):
                    if datatype == 'string':
                        if max_length:
                            column_args.insert(0, Column(record_key, String(max_length), primary_key=True))
                        else:
                            column_args.insert(0, Column(record_key, String, primary_key=True))
                    elif datatype == 'float':
                        column_args.insert(0, Column(record_key, Float, primary_key=True))
                    elif datatype == 'integer':
                        column_args.insert(0, Column(record_key, Integer, primary_key=True))
                else:
                    raise ValueError('Field "id" in record_schema must be a string, float or integer.')
            else:
                if datatype == 'boolean':
                    column_args.append(Column(record_key, Boolean))
                elif datatype == 'string':
                    if max_length:
                        column_args.append(Column(record_key, String(max_length)))
                    else:
                        column_args.append(Column(record_key, String))
                elif datatype == 'float':
                    column_args.append(Column(record_key, Float))
                elif datatype == 'integer':
                    column_args.append(Column(record_key, Integer))
                elif datatype == 'list':
                    column_args.append(Column(record_key, Binary))
        
        return column_args
1289186
def load_df(self, df):
    '''
    
    '''
    # self.__init__()
    self.reset()

    df_dict = {}
    df_dict['mat'] = deepcopy(df)
    # always define category colors if applicable when loading a df
    data_formats.df_to_dat(self, df_dict, define_cat_colors=True)
1358903
def map(cls, dict_entity):
        
        for key, value in dict_entity.items():
            if hasattr(cls, key):
                if isinstance(value, list):
                    _list = getattr(cls, key)
                    if isinstance(_list.expected_type, list):
                        for _dict in value:
                            _list.append(cls.map(_list.typeof(), _dict))
                elif isinstance(value, dict):
                    attr = getattr(cls, key)
                    instance = attr.expected_type()
                    Entity.map(instance, value)
                    setattr(cls, key, instance)
                else:
                    setattr(cls, key, value)
            else:
                setattr(cls, key, value)
354642
def encode_df(df, data_types={}):
    

    numbers = []
    categories = []
    datetimes = []
        
    for column, series in df.iteritems():
        if column in data_types:
            dtypes = data_types[column]
        else:
            dtypes = {}
        
        if 'datetime' in dtypes:
            index = pd.DatetimeIndex(pd.to_datetime(series.values))
            df[column] = index.astype(np.int64).astype(float)
            datetimes.append(column)
            
        elif any([dtype in dtypes for dtype in ['category', 'label']]):
            # If this can be treated as a category, encode it
            df = pd.get_dummies(df,columns=[column])
            categories.append(column)
                    
        elif 'numeric' in dtypes:
            df[column] = clean_numbers(df[column].values)
            df[column] = df[column].astype(float)
            numbers.append(column)
            
        else:
            df.drop(column,1,inplace=True)

    # Scale continuous columns
    scaler = MinMaxScaler()
    df[numbers + datetimes] = scaler.fit_transform(df[numbers + datetimes])
    
    return df, categories, numbers, datetimes