Help for making a DataFrame with Workbench CLI
435672
def help_dataframe_pe(self):
        
        help =  '%sMaking a DataFrame: %s how to make a dataframe from pe files' % (color.Yellow, color.Green)
        help += '\n\n%sPE Files Example (loading a directory):'  % (color.Green)
        help += '\n\t%s> load_sample /path/to/pe/bad [\'bad\', \'case_69\']' % (color.LightBlue)
        help += '\n\n\t%sSearch for all samples in the database that are pe files,'  % (color.Green)
        help += '\n\t%sthis command returns the sample_set containing the matching items'% (color.Green)
        help += '\n\t%s> my_exes = search([\'exe\'])' % (color.LightBlue)
        help += '\n\n\t%sRun workers on this sample_set:'  % (color.Green)
        help += '\n\t%s> pe_outputs = set_work_request(\'pe_features\', my_exes, [\'md5\', \'dense_features.*\', \'tags\'])' % (color.LightBlue)
        help += '\n\n\t%sMake a DataFrame:'  % (color.Green)
        help += '\n\t%s> pe_df = pd.DataFrame(pe_outputs) %s' % (color.LightBlue, color.Normal)
        help += '\n\t%s> pe_df.head() %s' % (color.LightBlue, color.Normal)
        help += '\n\t%s> pe_df = flatten_tags(pe_df) %s' % (color.LightBlue, color.Normal)
        help += '\n\t%s> pe_df.hist(\'check_sum\',\'tags\') %s' % (color.LightBlue, color.Normal)
        help += '\n\t%s> pe_df.bloxplot(\'check_sum\',\'tags\') %s' % (color.LightBlue, color.Normal)
        return help
66169
def from_dataframe(cls, name, df, indices, primary_key=None):
        """"""

        # ordered list (column_name, column_type) pairs
        column_types = []
        # which columns have nullable values
        nullable = set()

        # tag cached database by dataframe's number of rows and columns
        for column_name in df.columns:
            values = df[column_name]
            if values.isnull().any():
                nullable.add(column_name)
            column_db_type = db_type(values.dtype)
            column_types.append((column_name.replace(" ", "_"), column_db_type))

        def make_rows():
            return list(tuple(row) for row in df.values)

        return cls(
            name=name,
            column_types=column_types,
            make_rows=make_rows,
            indices=indices,
            nullable=nullable,
            primary_key=primary_key)
976664
def cli(
    paths,
    dbname,
    separator,
    quoting,
    skip_errors,
    replace_tables,
    table,
    extract_column,
    date,
    datetime,
    datetime_format,
    primary_key,
    fts,
    index,
    shape,
    filename_column,
    no_index_fks,
    no_fulltext_fks,
):
    
    # make plural for more readable code:
    extract_columns = extract_column
    del extract_column

    if extract_columns:
        click.echo("extract_columns={}".format(extract_columns))
    if dbname.endswith(".csv"):
        raise click.BadParameter("dbname must not end with .csv")
    if "." not in dbname:
        dbname += ".db"

    db_existed = os.path.exists(dbname)

    conn = sqlite3.connect(dbname)

    dataframes = []
    csvs = csvs_from_paths(paths)
    sql_type_overrides = None
    for name, path in csvs.items():
        try:
            df = load_csv(path, separator, skip_errors, quoting, shape)
            df.table_name = table or name
            if filename_column:
                df[filename_column] = name
                if shape:
                    shape += ",{}".format(filename_column)
            sql_type_overrides = apply_shape(df, shape)
            apply_dates_and_datetimes(df, date, datetime, datetime_format)
            dataframes.append(df)
        except LoadCsvError as e:
            click.echo("Could not load {}: {}".format(path, e), err=True)

    click.echo("Loaded {} dataframes".format(len(dataframes)))

    # Use extract_columns to build a column:(table,label) dictionary
    foreign_keys = {}
    for col in extract_columns:
        bits = col.split(":")
        if len(bits) == 3:
            foreign_keys[bits[0]] = (bits[1], bits[2])
        elif len(bits) == 2:
            foreign_keys[bits[0]] = (bits[1], "value")
        else:
            foreign_keys[bits[0]] = (bits[0], "value")

    # Now we have loaded the dataframes, we can refactor them
    created_tables = {}
    refactored = refactor_dataframes(
        conn, dataframes, foreign_keys, not no_fulltext_fks
    )
    for df in refactored:
        # This is a bit trickier because we need to
        # create the table with extra SQL for foreign keys
        if replace_tables and table_exists(conn, df.table_name):
            drop_table(conn, df.table_name)
        if table_exists(conn, df.table_name):
            df.to_sql(df.table_name, conn, if_exists="append", index=False)
        else:
            to_sql_with_foreign_keys(
                conn,
                df,
                df.table_name,
                foreign_keys,
                sql_type_overrides,
                primary_keys=primary_key,
                index_fks=not no_index_fks,
            )
            created_tables[df.table_name] = df
        if index:
            for index_defn in index:
                add_index(conn, df.table_name, index_defn)

    # Create FTS tables
    if fts:
        fts_version = best_fts_version()
        if not fts_version:
            conn.close()
            raise click.BadParameter(
                "Your SQLite version does not support any variant of FTS"
            )
        # Check that columns make sense
        for table, df in created_tables.items():
            for fts_column in fts:
                if fts_column not in df.columns:
                    raise click.BadParameter(
                        'FTS column "{}" does not exist'.format(fts_column)
                    )

        generate_and_populate_fts(conn, created_tables.keys(), fts, foreign_keys)

    conn.close()

    if db_existed:
        click.echo(
            "Added {} CSV file{} to {}".format(
                len(csvs), "" if len(csvs) == 1 else "s", dbname
            )
        )
    else:
        click.echo(
            "Created {} from {} CSV file{}".format(
                dbname, len(csvs), "" if len(csvs) == 1 else "s"
            )
        )
497460
def dataframe(self):
        """"""
        data = self.data
        channel_names = self.get_channel_names()
        return pd.DataFrame(data, columns=channel_names)
366190
def new_db_from_pandas(self, frame, table=None, data=None, load=True, **kwargs):
        

        from ..orm import Column
        # from dbexceptions import ConfigurationError

        # Create the table from the information in the data frame.
        with self.bundle.session:
            sch = self.bundle.schema
            t = sch.new_table(table)

            if frame.index.name:
                id_name = frame.index.name
            else:
                id_name = 'id'

            sch.add_column(t, id_name,
                           datatype=Column.convert_numpy_type(frame.index.dtype),
                           is_primary_key=True)

            for name, type_ in zip([row for row in frame.columns],
                                   [row for row in frame.convert_objects(convert_numeric=True,
                                                                         convert_dates=True).dtypes]):
                sch.add_column(t, name, datatype=Column.convert_numpy_type(type_))
                sch.write_schema()

        p = self.new_partition(table=table, data=data, **kwargs)

        if load:
            pk_name = frame.index.name
            with p.inserter(table) as ins:
                for i, row in frame.iterrows():
                    d = dict(row)
                    d[pk_name] = i

                    ins.insert(d)
        return p
1345233
def from_df(cls, df:DataFrame, cat_names:OptStrList=None, cont_names:OptStrList=None, procs=None, **kwargs)->'ItemList':
        ""
        return cls(items=range(len(df)), cat_names=cat_names, cont_names=cont_names, procs=procs, inner_df=df.copy(), **kwargs)
1097959
public static void main(String[] args) {
    SparkSession spark = SparkSession
      .builder()
      .appName("Java Spark SQL user-defined DataFrames aggregation example")
      .getOrCreate();

    // $example on:untyped_custom_aggregation$
    // Register the function to access it
    spark.udf().register("myAverage", new MyAverage());

    Dataset<Row> df = spark.read().json("examples/src/main/resources/employees.json");
    df.createOrReplaceTempView("employees");
    df.show();
    // +-------+------+
    // |   name|salary|
    // +-------+------+
    // |Michael|  3000|
    // |   Andy|  4500|
    // | Justin|  3500|
    // |  Berta|  4000|
    // +-------+------+

    Dataset<Row> result = spark.sql("SELECT myAverage(salary) as average_salary FROM employees");
    result.show();
    // +--------------+
    // |average_salary|
    // +--------------+
    // |        3750.0|
    // +--------------+
    // 

    spark.stop();
  }
1000608
def to_excel(self, *args):
        
        path = os.getcwd()
        fname = self.fname.replace(".tpl", "_tpl") + ".xlsx"
        idxs = self.filter_trends("")
        for idx in idxs:
            self.extract(idx)
        data_df = pd.DataFrame(self.data)
        data_df.columns = self.label.values()
        data_df.insert(0, "Time [s]", self.time)
        if len(args) > 0 and args[0] != "":
            path = args[0]
            if os.path.exists(path) == False:
                os.mkdir(path)
            data_df.to_excel(path + os.sep + fname)
        else:
            data_df.to_excel(self.path + os.sep + fname)
1015351
def add_dataframe(df, name, pkg=None, description=''):
    
    from warnings import warn
    from metapack.cli.core import alt_col_name, type_map
    import numpy as np

    if name is None or df is None:
        warn("Did not find dataframe for reference '{}' ".format(ref))
        return

    pkg = pkg or open_source_package()

    resource_ref = 'file:' + get_notebook_rel_path(pkg) + '#' + name

    t = pkg.find_first('Root.Datafile', value=resource_ref)
    col_props = {}

    if t:
        print("Datafile exists for url '{}', deleting".format(resource_ref))

        if t.schema_term:
            col_props = { c['name']:c for c in t.columns()}
            pkg.remove_term(t.schema_term)

        pkg.remove_term(t)

    t = pkg['Resources'].new_term('Root.Datafile', resource_ref, name=name, description=description)

    st = pkg['Schema'].new_term('Table', t.schema_name, description=description)

    for i, name in enumerate(df.columns):

        props = col_props.get(name,{})

        try:
            native_type = type(np.asscalar(df[name].dtype.type(0))).__name__
        except ValueError:
            native_type = df[name].dtype.name
        except AttributeError:
            native_type = type(df[name][0]).__name__

        for pn in 'datatype name pos header'.split():
            if pn in props:
                del props[pn]

        if 'altname' in props:
            altname = props['altname']
            del props['altname']
        else:
            raw_alt_name = alt_col_name(name, i)
            altname = raw_alt_name if raw_alt_name != name else ''

        col = df[name]

        if hasattr(col, 'description'):  # custom property
            props['description'] = col.description

        t = st.new_child('Column', name,
                            datatype=type_map.get(native_type, native_type),
                            altname=altname,
                            **props)


    pkg.write_csv()
808060
def to_dataframe(self, columns=None):
    """"""
    if not self.is_value_set:
      raise ValueError('Value must be set before converting to a DataFrame.')
    if not pandas:
      raise RuntimeError('Install pandas to convert to pandas.DataFrame')
    return pandas.DataFrame.from_records(self.value, columns=columns)
581978
def create_matrixes(chip, input, df, args):
    # type: (Iterable[pd.DataFrame], Iterable[pd.DataFrame], pd.DataFrame, Namespace) -> List[pd.DataFrame]
    ""

    genome = args.chromosome_sizes

    chip = put_dfs_in_chromosome_dict(chip)
    input = put_dfs_in_chromosome_dict(input)
    all_chromosomes = natsorted(set(list(chip.keys()) + list(input.keys())))

    # print("df1\n", df, file=sys.stderr)
    islands = enriched_bins(df, args)
    # print("islands1\n", islands, file=sys.stderr)


    logging.info("Creating matrixes from count data.")
    dfms = Parallel(n_jobs=args.number_cores)(delayed(_create_matrixes)(
        chromosome, chip, input, islands, genome[chromosome],
        args.window_size) for chromosome in all_chromosomes)

    return dfms
7337
def to_dataframe(products):
        
        try:
            import pandas as pd
        except ImportError:
            raise ImportError("to_dataframe requires the optional dependency Pandas.")

        return pd.DataFrame.from_dict(products, orient='index')
11371
def _construct_df(pdb_lines):
        """"""
        valids = tuple(pdb_records.keys())
        line_lists = {r: [] for r in valids}
        line_lists['OTHERS'] = []
        for line_num, line in enumerate(pdb_lines):
            if line.strip():
                if line.startswith(valids):
                    record = line[:6].rstrip()
                    line_ele = ['' for _ in range(len(
                        pdb_records[record]) + 1)]
                    for idx, ele in enumerate(pdb_records[record]):
                        line_ele[idx] = (line[ele['line'][0]:ele['line'][1]]
                                         .strip())
                    line_ele[-1] = line_num
                    line_lists[record].append(line_ele)
                else:
                    line_lists['OTHERS'].append([line[:6].rstrip(),
                                                line[6:-1].rstrip(), line_num])

        dfs = {}
        for r in line_lists.items():
            df = pd.DataFrame(r[1], columns=[c['id'] for c in
                                             pdb_records[r[0]]] + ['line_idx'])
            for c in pdb_records[r[0]]:
                try:
                    df[c['id']] = df[c['id']].astype(c['type'])
                except ValueError:
                    # expect ValueError if float/int columns are empty strings
                    df[c['id']] = pd.Series(np.nan, index=df.index)

            dfs[r[0]] = df
        return dfs
549561
def save_dataframes(self, outdir, prefix='df_'):
        
        # Get list of attributes that start with "df_"
        dfs = list(filter(lambda x: x.startswith(prefix), dir(self)))

        counter = 0
        for df in dfs:
            outpath = ssbio.utils.outfile_maker(inname=df, outext='.csv', outdir=outdir)
            my_df = getattr(self, df)
            if not isinstance(my_df, pd.DataFrame):
                raise TypeError('{}: object is not a Pandas DataFrame'.format(df))

            if my_df.empty:
                log.debug('{}: empty dataframe, not saving'.format(df))
            else:
                my_df.to_csv(outpath)
                log.debug('{}: saved dataframe'.format(outpath))
                counter += 1

        log.debug('Saved {} dataframes at {}'.format(counter, outdir))
965980
def to_dataframe(self, **kwargs):
        """"""
        return pandas.io.parsers.read_csv(self.path, sep=self.d, **kwargs)
281916
def to_dataframe(self):
        
        keys = self.data[0].keys()
        column_list =[]
        for k in keys:
            key_list = []
            for i in xrange(0,len(self.data)):
                key_list.append(self.data[i][k])
            column_list.append(key_list)
        df = DataFrame(np.asarray(column_list).transpose(), columns=keys)
        for i in xrange(0,df.shape[1]):
            if is_number(df.iloc[:,i]):
                df.iloc[:,i] = df.iloc[:,i].astype(float)
        return df
608195
def __build_dataframe(self, timing, project_name=None, org_name=None):
        
        date_list = []
        uuid_list = []
        name_list = []
        contribs_list = []
        latest_ts_list = []
        logger.debug(self.__log_prefix + " timing: " + timing.key_as_string)

        for author in timing[self.AUTHOR_UUID].buckets:
            latest_ts_list.append(timing[self.LATEST_TS].value_as_string)
            date_list.append(timing.key_as_string)
            uuid_list.append(author.key)
            if author[self.AUTHOR_NAME] and author[self.AUTHOR_NAME].buckets \
                    and len(author[self.AUTHOR_NAME].buckets) > 0:
                name_list.append(author[self.AUTHOR_NAME].buckets[0].key)
            else:
                name_list.append("Unknown")
            contribs_list.append(author[self.CONTRIBUTIONS].value)

        df = pandas.DataFrame()
        df[self.TIMEFRAME] = date_list
        df[self.AUTHOR_UUID] = uuid_list
        df[self.AUTHOR_NAME] = name_list
        df[self.CONTRIBUTIONS] = contribs_list
        df[self.TIMESTAMP] = latest_ts_list

        if not project_name:
            project_name = "_Global_"
        df[self.PROJECT] = project_name

        if not org_name:
            org_name = "_Global_"
        df[self.AUTHOR_ORG] = org_name

        return df