Dumps in df in chunks to avoid crashes.
493876
def create_df_file_with_query(self, query, output):
        
        chunk_size = 100000
        offset = 0
        data = defaultdict(lambda : defaultdict(list))

        with open(output, 'wb') as outfile:
            query = query.replace(';', '')
            query += """ LIMIT {chunk_size} OFFSET {offset};"""
            while True:
                print(offset)
                query = query.format(
                    chunk_size=chunk_size,
                    offset=offset
                )
                df = pd.read_sql(query, self.engine)
                pickle.dump(df, outfile)
                offset += chunk_size
                if len(df) < chunk_size:
                    break
            outfile.close()
1004221
def save_hdf(self,filename,path=''):
        
        self.dataframe.to_hdf(filename,'{}/df'.format(path))
1362736
def store(self, loc, df):
        

        path = "%s.%s" % (self._root / "processed" / loc, FILE_EXTENSION)
        WRITE_DF(df, path, **WRITE_DF_OPTS)
        self._cache[loc] = df
832535
def write_dataframe(rows, encoding=ENCODING, dialect=DIALECT, **kwargs):
    """"""
    global pandas
    if pandas is None:  # pragma: no cover
        import pandas
    with contextlib.closing(CsvBuffer()) as fd:
        write_csv(fd, rows, encoding, dialect)
        fd.seek(0)
        df = read_csv(pandas, fd, encoding, dialect, kwargs)
    return df
983991
def flatten(self, df, column_name):
        """"""
        _exp_list = [[md5, x] for md5, value_list in zip(df['md5'], df[column_name]) for x in value_list]
        return pd.DataFrame(_exp_list, columns=['md5',column_name])
783530
def df2unstack(df,coln='columns',idxn='index',col='value'):
    
    return dmap2lin(df,idxn=idxn,coln=coln,colvalue_name=col)
131761
def dumps(columns):
    '''
    
    '''
    fp = BytesIO()
    dump(columns, fp)
    fp.seek(0)
    return fp.read()
134668
def safe_dump(data, stream=None, **kwds):
    
    return dump_all([data], stream, Dumper=SafeDumper, **kwds)
1124395
def dump(self, f):
        
        ret = libxml2mod.xmlDocDump(f, self._o)
        return ret
1368861
def write_r(df, f, sep=",", index_join="@", columns_join="."):
    

    df = df.copy()
    df.index = ["@".join([str(s) for s in v]) for v in df.index.values]
    df.columns = [".".join([str(s) for s in v]) for v in df.index.values]
    df.to_csv(f, sep=sep)
595189
def _pre_dump(cls):
        """"""
        shutil.rmtree(cls.outdir, ignore_errors=True)
        os.makedirs(cls.outdir)
        super(PlotMetric, cls)._pre_dump()
316724
def edge_val_dump(self):
        """"""
        self._flush_edge_val()
        for (
                graph, orig, dest, idx, key, branch, turn, tick, value
        ) in self.sql('edge_val_dump'):
            yield (
                self.unpack(graph),
                self.unpack(orig),
                self.unpack(dest),
                idx,
                self.unpack(key),
                branch,
                turn,
                tick,
                self.unpack(value)
            )
1136924
def dumpfile(self, fd):
        

        self.start()
        dump = DumpFile(fd)
        self.queue.append(dump)
1296972
def dump_bulk(cls, parent=None, keep_ids=True):
        """"""

        serializable_cls = cls._get_serializable_model()
        if (
                parent and serializable_cls != cls and
                parent.__class__ != serializable_cls
        ):
            parent = serializable_cls.objects.get(pk=parent.pk)

        # a list of nodes: not really a queryset, but it works
        objs = serializable_cls.get_tree(parent)

        ret, lnk = [], {}
        for node, pyobj in zip(objs, serializers.serialize('python', objs)):
            depth = node.get_depth()
            # django's serializer stores the attributes in 'fields'
            fields = pyobj['fields']
            del fields['parent']

            # non-sorted trees have this
            if 'sib_order' in fields:
                del fields['sib_order']

            if 'id' in fields:
                del fields['id']

            newobj = {'data': fields}
            if keep_ids:
                newobj['id'] = pyobj['pk']

            if (not parent and depth == 1) or\
               (parent and depth == parent.get_depth()):
                ret.append(newobj)
            else:
                parentobj = lnk[node.parent_id]
                if 'children' not in parentobj:
                    parentobj['children'] = []
                parentobj['children'].append(newobj)
            lnk[node.pk] = newobj
        return ret
735094
def pad_chunk_columns(chunk):
    """"""
    columns = set()
    for record in chunk:
        columns.update(record.keys())
    for record in chunk:
        for column in columns:
            record.setdefault(column, None)
    return chunk
125156
def clip(self, lower=None, upper=None):
    '''
    
    '''
    df = self.export_df()
    df = df.clip(lower=lower, upper=upper)
    self.load_df(df)
549561
def save_dataframes(self, outdir, prefix='df_'):
        
        # Get list of attributes that start with "df_"
        dfs = list(filter(lambda x: x.startswith(prefix), dir(self)))

        counter = 0
        for df in dfs:
            outpath = ssbio.utils.outfile_maker(inname=df, outext='.csv', outdir=outdir)
            my_df = getattr(self, df)
            if not isinstance(my_df, pd.DataFrame):
                raise TypeError('{}: object is not a Pandas DataFrame'.format(df))

            if my_df.empty:
                log.debug('{}: empty dataframe, not saving'.format(df))
            else:
                my_df.to_csv(outpath)
                log.debug('{}: saved dataframe'.format(outpath))
                counter += 1

        log.debug('Saved {} dataframes at {}'.format(counter, outdir))