Save all attributes that start with "df" into a specified directory.

        Args:
            outdir (str): Path to output directory
            prefix (str): Prefix that dataframe attributes start with
549561
def save_dataframes(self, outdir, prefix='df_'):
        
        # Get list of attributes that start with "df_"
        dfs = list(filter(lambda x: x.startswith(prefix), dir(self)))

        counter = 0
        for df in dfs:
            outpath = ssbio.utils.outfile_maker(inname=df, outext='.csv', outdir=outdir)
            my_df = getattr(self, df)
            if not isinstance(my_df, pd.DataFrame):
                raise TypeError('{}: object is not a Pandas DataFrame'.format(df))

            if my_df.empty:
                log.debug('{}: empty dataframe, not saving'.format(df))
            else:
                my_df.to_csv(outpath)
                log.debug('{}: saved dataframe'.format(outpath))
                counter += 1

        log.debug('Saved {} dataframes at {}'.format(counter, outdir))
754716
def write_table_to_file(self, dtype, custom_name=None, append=False, dir_path=None):
        
        if custom_name:
            fname = custom_name
        else:
            fname = self.filenames[dtype]
        if not dir_path:
            dir_path=self.directory
        if dtype in self.tables:
            write_df = self.remove_names(dtype)
            outfile = self.tables[dtype].write_magic_file(custom_name=fname,
                                                          dir_path=dir_path,
                                                          append=append, df=write_df)
        return outfile
324971
def convert_dirs(base_dir, hdf_name, complib=None, complevel=0):
    
    print('Converting directories in {}'.format(base_dir))

    dirs = glob.glob(os.path.join(base_dir, '*'))
    dirs = {d for d in dirs if os.path.basename(d) in DIRECTORIES}
    if not dirs:
        raise RuntimeError('No direcotries found matching known data.')

    store = pd.HDFStore(
        hdf_name, mode='w', complevel=complevel, complib=complib)

    for dirpath in dirs:
        dirname = os.path.basename(dirpath)

        print(dirname)
        df = cache_to_df(dirpath)

        if dirname == 'travel_data':
            keys = ['from_zone_id', 'to_zone_id']
        elif dirname == 'annual_employment_control_totals':
            keys = ['sector_id', 'year', 'home_based_status']
        elif dirname == 'annual_job_relocation_rates':
            keys = ['sector_id']
        elif dirname == 'annual_household_control_totals':
            keys = ['year']
        elif dirname == 'annual_household_relocation_rates':
            keys = ['age_of_head_max', 'age_of_head_min',
                    'income_min', 'income_max']
        elif dirname == 'building_sqft_per_job':
            keys = ['zone_id', 'building_type_id']
        elif dirname == 'counties':
            keys = ['county_id']
        elif dirname == 'development_event_history':
            keys = ['building_id']
        elif dirname == 'target_vacancies':
            keys = ['building_type_id', 'year']
        else:
            keys = [dirname[:-1] + '_id']

        if dirname != 'annual_household_relocation_rates':
            df = df.set_index(keys)

        for colname in df.columns:
            if df[colname].dtype == np.float64:
                df[colname] = df[colname].astype(np.float32)
            elif df[colname].dtype == np.int64:
                df[colname] = df[colname].astype(np.int32)
            else:
                df[colname] = df[colname]

        df.info()
        print(os.linesep)
        store.put(dirname, df)

    store.close()
774034
def main(net, df=None):
  '''
  
  '''
  if df is None:
    df = net.export_df()

  # rows
  #############
  rows = df.index.tolist()
  if type(rows[0]) is str:

    if len(rows) != len(list(set(rows))):
      new_rows = add_index_list(rows)
      df.index = new_rows

  elif type(rows[0]) is tuple:

    row_names = []
    for inst_row in rows:
      row_names.append(inst_row[0])

    if len(row_names) != len(list(set(row_names))):
      row_names = add_index_list(row_names)

      # add back to tuple
      new_rows = []
      for inst_index in range(len(rows)):
        inst_row = rows[inst_index]
        new_row = list(inst_row)
        new_row[0] = row_names[inst_index]
        new_row = tuple(new_row)
        new_rows.append(new_row)

      df.index = new_rows

  # cols
  #############
  cols = df.columns.tolist()
  if type(cols[0]) is str:

    # list column names
    if len(cols) != len(list(set(cols))):
      new_cols = add_index_list(cols)
      df.columns = new_cols

  elif type(cols[0]) is tuple:

    col_names = []
    for inst_col in cols:
      col_names.append(inst_col[0])

    if len(col_names) != len(list(set(col_names))):
      col_names = add_index_list(col_names)

      # add back to tuple
      new_cols = []
      for inst_index in range(len(cols)):
        inst_col = cols[inst_index]
        new_col = list(inst_col)
        new_col[0] = col_names[inst_index]
        new_col = tuple(new_col)
        new_cols.append(new_col)

      df.columns = new_cols

  # return dataframe with unique names
  return df
1005230
def prepare_output(d):
    """"""
    outDir = os.path.join(d, 'inorder')
    if not os.path.exists(outDir):
        os.mkdir(outDir)
    for f in os.listdir(outDir):
        f = os.path.join(outDir, f)
        if os.path.islink(f):
            os.remove(f)
    return outDir
142857
def init(*, output_dir=FS_DEFAULT_OUTPUT_DIR, dry_run=False, **kwargs):
    
    # Output directory
    global _output_dir
    _output_dir = output_dir

    # Dry run mode
    global _dry_run
    _dry_run = dry_run

    # Type checks
    utils.chkstr(_output_dir, 'output_dir')

    # log the thing
    log.msg("Output directory will be: {output_dir}"
            .format(output_dir=_output_dir))

    # Create output directory if it does not exist
    if not os.path.exists(_output_dir):
        log.msg_warn("Output path '{out_dir}' does not exist, creating it ..."
                     .format(out_dir=_output_dir))
        if not _dry_run:
            # create the actual root output directory
            os.makedirs(_output_dir)
            # set folder permissions to 0770
            os.chmod(_output_dir, stat.S_IRWXU | stat.S_IRWXG)
661323
def save_attributes(self):
        '''
        
        '''
        if not self.saveable():
            raise AttributeError("Cannot save attribute file without a valid file")
        if self._read_only:
            raise AttributeError("Cannot save read-only data")
        if not self._db_closed:
            self._fd.clear()
            for attr_name in self._saved_attrs:
                self._fd[attr_name] = getattr(self._target, attr_name)
1305848
def dump(self, directory=''):
        
        with self.no_unpicklable_properties():
            file_name = os.path.join(directory, 'job-instance.pickle')
            if self.__module__ == '__main__':
                d = pickle.dumps(self)
                module_name = os.path.basename(sys.argv[0]).rsplit('.', 1)[0]
                d = d.replace(b'(c__main__', "(c" + module_name)
                open(file_name, "wb").write(d)

            else:
                pickle.dump(self, open(file_name, "wb"))
1281453
def exporter(directory, method, datasets):
    """"""
    if method.lower() == 'json':
        # Convert json_dict to a JSON styled string
        json_string = json.dumps(datasets, indent=4)
        savefile = open('{}/exported.json'.format(directory), 'w+')
        savefile.write(json_string)
        savefile.close()

    if method.lower() == 'csv':
        with open('{}/exported.csv'.format(directory), 'w+') as csvfile:
            csv_writer = csv.writer(
                csvfile, delimiter=',', quoting=csv.QUOTE_MINIMAL)
            for key, values in datasets.items():
                if values is None:
                    csv_writer.writerow([key])
                else:
                    csv_writer.writerow([key] + values)
        csvfile.close()
1073163
def prepare_outdir(outdir):
    
    if outdir:
        outdir = os.path.expanduser(outdir)
        if not os.path.isdir(outdir):
            try:
                os.makedirs(outdir)
            except os.error as e:
                raise JobExecutionError('Failed to create outdir "{}".\n{}'.format(outdir, str(e)))
1368861
def write_r(df, f, sep=",", index_join="@", columns_join="."):
    

    df = df.copy()
    df.index = ["@".join([str(s) for s in v]) for v in df.index.values]
    df.columns = [".".join([str(s) for s in v]) for v in df.index.values]
    df.to_csv(f, sep=sep)
695962
def _to_dir(self, file_generator):
        
        for f in file_generator:
            outpath = self._output_path(f.source, file_generator.to_format)

            if not os.path.exists(os.path.dirname(outpath)):
                os.makedirs(os.path.dirname(outpath))

            with open(outpath, mode="w") as outfile:
                f.write(outfile, file_generator.to_format)
1174567
def sample_out_dir(self):
        
        if self._sample_out_dir is None:
            self._sample_out_dir = os.path.join(
                self.output_dir, self._sample_name
            )
        return self._sample_out_dir
408689
def df_to_dat(self, df, define_cat_colors=False):
    '''
    
    '''
    data_formats.df_to_dat(self, df, define_cat_colors)
871218
def merge_dfs(dfs, resample=None, do_mean=False, do_sum=False, do_min=False, do_max=False):
    
    if len(dfs) == 0:
        raise Exception("No dataframes provided")
    df = dfs.values()[0]
    name = dfs.keys()[0]
    df.columns = map(lambda x: name+"_"+x if not x.startswith(name) else x, df.columns)
    if resample is not None:
        df = df.resample(resample)
        if do_mean: df = df.mean()
        elif do_sum: df = df.sum()
        elif do_min: df = df.min()
        elif do_max: df = df.max()
        else: df = df.mean()
    if len(dfs) > 1:
        for name, newdf in dfs.items()[1:]:
            if resample is not None:
                newdf = newdf.resample(resample)
                if do_mean: newdf = newdf.mean()
                elif do_sum: newdf = newdf.sum()
                elif do_min: newdf = newdf.min()
                elif do_max: newdf = newdf.max()
                else: newdf = newdf.mean()
            newdf.columns = map(lambda x: name+"_"+x if not x.startswith(name) else x, newdf.columns)
            df = df.merge(newdf, left_index=True, right_index=True, how='outer')
    return df
604003
def df(self, src):
        ''''''
        return self._getStdOutCmd([self._hadoop_cmd, 'fs', '-df', self._full_hdfs_path(src)], True)
247077
def _dir_out(self):
        """"""
        return os.path.join(self.proj.direc_out, self.proj.name,
                            self.model.name, self.run.name, self.name)