A wrapper for the df shell command.
493399
def df(unit = 'GB'):
    ''''''
    details = {}
    headers = ['Filesystem', 'Type', 'Size', 'Used', 'Available', 'Capacity', 'MountedOn']
    n = len(headers)

    unit = df_conversions[unit]
    p = subprocess.Popen(args = ['df', '-TP'], stdout = subprocess.PIPE) # -P prevents line wrapping on long filesystem names
    stdout, stderr = p.communicate()

    lines = stdout.split("\n")
    lines[0] = lines[0].replace("Mounted on", "MountedOn").replace("1K-blocks", "Size").replace("1024-blocks", "Size")
    assert(lines[0].split() == headers)

    lines = [l.strip() for l in lines if l.strip()]
    for line in lines[1:]:
        tokens = line.split()
        if tokens[0] == 'none': # skip uninteresting entries
            continue

        assert(len(tokens) == n)
        d = {}
        for x in range(1, len(headers)):
            d[headers[x]] = tokens[x]
        d['Size'] = float(d['Size']) / unit
        assert(d['Capacity'].endswith("%"))
        d['Use%'] = d['Capacity']
        d['Used'] = float(d['Used']) / unit
        d['Available'] = float(d['Available']) / unit
        d['Using'] = 100*(d['Used']/d['Size']) # same as Use% but with more precision

        if d['Type'].startswith('ext'):
            pass
            d['Using'] += 5 # ext2, ext3, and ext4 reserve 5% by default
        else:
            ext3_filesystems = ['ganon:', 'kortemmelab:', 'albana:']
            for e3fs in ext3_filesystems:
                if tokens[0].find(e3fs) != -1:
                    d['Using'] += 5 # ext3 reserves 5%
                    break

        details[tokens[0]] = d

    return details
1107009
def sh2(cmd):
    """"""
    p = Popen(cmd, stdout=PIPE, shell=True, env=sub_environment())
    out = p.communicate()[0]
    retcode = p.returncode
    if retcode:
        raise CalledProcessError(retcode, cmd)
    else:
        return out.rstrip()
671744
def T(u, dfs_data):
    """"""
    return list(set([w for v in S_star(u, dfs_data) for w in A(v, dfs_data)]))
976664
def cli(
    paths,
    dbname,
    separator,
    quoting,
    skip_errors,
    replace_tables,
    table,
    extract_column,
    date,
    datetime,
    datetime_format,
    primary_key,
    fts,
    index,
    shape,
    filename_column,
    no_index_fks,
    no_fulltext_fks,
):
    
    # make plural for more readable code:
    extract_columns = extract_column
    del extract_column

    if extract_columns:
        click.echo("extract_columns={}".format(extract_columns))
    if dbname.endswith(".csv"):
        raise click.BadParameter("dbname must not end with .csv")
    if "." not in dbname:
        dbname += ".db"

    db_existed = os.path.exists(dbname)

    conn = sqlite3.connect(dbname)

    dataframes = []
    csvs = csvs_from_paths(paths)
    sql_type_overrides = None
    for name, path in csvs.items():
        try:
            df = load_csv(path, separator, skip_errors, quoting, shape)
            df.table_name = table or name
            if filename_column:
                df[filename_column] = name
                if shape:
                    shape += ",{}".format(filename_column)
            sql_type_overrides = apply_shape(df, shape)
            apply_dates_and_datetimes(df, date, datetime, datetime_format)
            dataframes.append(df)
        except LoadCsvError as e:
            click.echo("Could not load {}: {}".format(path, e), err=True)

    click.echo("Loaded {} dataframes".format(len(dataframes)))

    # Use extract_columns to build a column:(table,label) dictionary
    foreign_keys = {}
    for col in extract_columns:
        bits = col.split(":")
        if len(bits) == 3:
            foreign_keys[bits[0]] = (bits[1], bits[2])
        elif len(bits) == 2:
            foreign_keys[bits[0]] = (bits[1], "value")
        else:
            foreign_keys[bits[0]] = (bits[0], "value")

    # Now we have loaded the dataframes, we can refactor them
    created_tables = {}
    refactored = refactor_dataframes(
        conn, dataframes, foreign_keys, not no_fulltext_fks
    )
    for df in refactored:
        # This is a bit trickier because we need to
        # create the table with extra SQL for foreign keys
        if replace_tables and table_exists(conn, df.table_name):
            drop_table(conn, df.table_name)
        if table_exists(conn, df.table_name):
            df.to_sql(df.table_name, conn, if_exists="append", index=False)
        else:
            to_sql_with_foreign_keys(
                conn,
                df,
                df.table_name,
                foreign_keys,
                sql_type_overrides,
                primary_keys=primary_key,
                index_fks=not no_index_fks,
            )
            created_tables[df.table_name] = df
        if index:
            for index_defn in index:
                add_index(conn, df.table_name, index_defn)

    # Create FTS tables
    if fts:
        fts_version = best_fts_version()
        if not fts_version:
            conn.close()
            raise click.BadParameter(
                "Your SQLite version does not support any variant of FTS"
            )
        # Check that columns make sense
        for table, df in created_tables.items():
            for fts_column in fts:
                if fts_column not in df.columns:
                    raise click.BadParameter(
                        'FTS column "{}" does not exist'.format(fts_column)
                    )

        generate_and_populate_fts(conn, created_tables.keys(), fts, foreign_keys)

    conn.close()

    if db_existed:
        click.echo(
            "Added {} CSV file{} to {}".format(
                len(csvs), "" if len(csvs) == 1 else "s", dbname
            )
        )
    else:
        click.echo(
            "Created {} from {} CSV file{}".format(
                dbname, len(csvs), "" if len(csvs) == 1 else "s"
            )
        )
178548
def main(argv=None):
    """"""
    args = docopt(__doc__, version=pycdstar.__version__, argv=argv, options_first=True)
    subargs = [args['<command>']] + args['<args>']

    if args['<command>'] in ['help', None]:
        cmd = None
        if len(subargs) > 1:
            cmd = COMMANDS.get(subargs[1])
        if cmd:
            print(cmd.__doc__)
        else:
            print(__doc__)
        return 0

    cmd = COMMANDS.get(args['<command>'])
    if not cmd:
        print('unknown command')
        print(__doc__)
        return 0

    cfg = Config(**dict(
        cfg=args.pop('--cfg', None),
        url=args.pop('--service', None),
        user=args.pop('--user', None),
        password=args.pop('--password', None)))

    try:
        res = cmd(
            Cdstar(cfg=cfg),
            docopt(cmd.__doc__, argv=subargs),
            verbose=args.get('--verbose'))
        if isinstance(res, types.GeneratorType):
            res = list(res)
        if isinstance(res, list):
            for line in res:
                print(line)
            res = 0
        return res or 0
    except:  # noqa: E722; # pragma: no cover
        # FIXME: log exception!
        return 256
1266881
def from_df(cls, df:DataFrame, path:PathOrStr, cols:IntsOrStrs=0, folder:PathOrStr=None, suffix:str='', **kwargs)->'ItemList':
        ""
        suffix = suffix or ''
        res = super().from_df(df, path=path, cols=cols, **kwargs)
        pref = f'{res.path}{os.path.sep}'
        if folder is not None: pref += f'{folder}{os.path.sep}'
        res.items = np.char.add(np.char.add(pref, res.items.astype(str)), suffix)
        return res
686130
def cli(ctx, env):
    """"""
    env.out("Welcome to the SoftLayer shell.")
    env.out("")

    formatter = formatting.HelpFormatter()
    commands = []
    shell_commands = []
    for name in cli_core.cli.list_commands(ctx):
        command = cli_core.cli.get_command(ctx, name)
        if command.short_help is None:
            command.short_help = command.help
        details = (name, command.short_help)
        if name in dict(routes.ALL_ROUTES):
            shell_commands.append(details)
        else:
            commands.append(details)

    with formatter.section('Shell Commands'):
        formatter.write_dl(shell_commands)

    with formatter.section('Commands'):
        formatter.write_dl(commands)

    for line in formatter.buffer:
        env.out(line, newline=False)
539538
def main():
    """"""
    parser = argparse.ArgumentParser(description='A Python tool that automatically cleans data sets and readies them for analysis')

    parser.add_argument('INPUT_FILENAME', type=str, help='File name of the data file to clean')

    parser.add_argument('-cv', action='store', dest='CROSS_VAL_FILENAME', default=None,
                         type=str, help='File name for the validation data set if performing cross-validation')

    parser.add_argument('-o', action='store', dest='OUTPUT_FILENAME', default=None,
                        type=str, help='Data file to output the cleaned data set to')

    parser.add_argument('-cvo', action='store', dest='CV_OUTPUT_FILENAME', default=None,
                        type=str, help='Data file to output the cleaned cross-validation data set to')

    parser.add_argument('-is', action='store', dest='INPUT_SEPARATOR', default='\t',
                        type=str, help='Column separator for the input file(s) (default: \\t)')
                    
    parser.add_argument('-os', action='store', dest='OUTPUT_SEPARATOR', default='\t',
                        type=str, help='Column separator for the output file(s) (default: \\t)')

    parser.add_argument('--drop-nans', action='store_true', dest='DROP_NANS', default=False,
                        help='Drop all rows that have a NaN in any column (default: False)')
                        
    parser.add_argument('--ignore-update-check', action='store_true', dest='IGNORE_UPDATE_CHECK', default=False,
                        help='Do not check for the latest version of datacleaner (default: False)')

    parser.add_argument('--version', action='version', version='datacleaner v{version}'.format(version=__version__))

    args = parser.parse_args()

    input_data = pd.read_csv(args.INPUT_FILENAME, sep=args.INPUT_SEPARATOR)
    if args.CROSS_VAL_FILENAME is None:
        clean_data = autoclean(input_data, drop_nans=args.DROP_NANS, ignore_update_check=args.IGNORE_UPDATE_CHECK)
        if args.OUTPUT_FILENAME is None:
            print('Cleaned data set:')
            print(clean_data)
            print('')
            print('If you cannot view the entire data set, output it to a file instead. '
                  'Type datacleaner --help for more information.')
        else:
            clean_data.to_csv(args.OUTPUT_FILENAME, sep=args.OUTPUT_SEPARATOR, index=False)
    else:
        if args.OUTPUT_FILENAME is not None and args.CV_OUTPUT_FILENAME is None:
            print('You must specify both output file names. Type datacleaner --help for more information.')
            return
    
        cross_val_data = pd.read_csv(args.CROSS_VAL_FILENAME, sep=args.INPUT_SEPARATOR)
        clean_training_data, clean_testing_data = autoclean_cv(input_data, cross_val_data,
                                                               drop_nans=args.DROP_NANS,
                                                               ignore_update_check=args.IGNORE_UPDATE_CHECK)

        if args.OUTPUT_FILENAME is None:
            print('Cleaned training data set:')
            print(clean_training_data)
            print('')
            print('Cleaned testing data set:')
            print(clean_testing_data)
            print('')
            print('If you cannot view the entire data set, output it to a file instead. '
                  'Type datacleaner --help for more information.')
        else:
            clean_training_data.to_csv(args.OUTPUT_FILENAME, sep=args.OUTPUT_SEPARATOR, index=False)
            clean_testing_data.to_csv(args.OUTPUT_FILENAME, sep=args.OUTPUT_SEPARATOR, index=False)
415112
def preprocess(df):
    """"""

    def process_row(idx_row):
        idx, row = idx_row

        # press-press latency
        tau = 1000 * row[4::3].astype(float).values
        tau = np.r_[np.median(tau), tau]

        # duration
        duration = 1000 * row[3::3].astype(float).values

        keyname = list('.tie5Roanl') + ['enter']

        return pd.DataFrame.from_items([
            ('user', [row['subject']] * 11),
            ('session', [row['sessionIndex'] * 100 + row['rep']] * 11),
            ('tau', tau),
            ('duration', duration),
            ('event', keyname)
        ])

    df = pd.concat(map(process_row, df.iterrows())).set_index(['user', 'session'])
    return df
1100973
def _apply_criteria(df, criteria, **kwargs):
    """"""
    idxs = []
    for var, check in criteria.items():
        _df = df[df['variable'] == var]
        for group in _df.groupby(META_IDX):
            grp_idxs = _check_rows(group[-1], check, **kwargs)
            idxs.append(grp_idxs)
    df = df.loc[itertools.chain(*idxs)]
    return df
526857
def col_mod(df,col_name,func,*args,**kwargs):
    
    backup = df[col_name].copy()
    try:
        return_val = func(df,col_name,*args,**kwargs)
        if return_val is not None:
            set_col(df,col_name,return_val)
    except:
        df[col_name] = backup
504813
def cmdargv(self):
        '''
        
        '''

        argv = []
        while self.more():

            # cmdargv *requires* whitespace
            if not self.ignore(whitespace):
                break

            # if we hit a | or a } we're done
            if self.nextstr('|'):
                break

            if self.nextstr('}'):
                break

            if not self.nextstr('{'):
                valu = self.cmdvalu()
                argv.append(valu)
                continue

            start = self.offs
            self.subquery()

            text = self.text[start:self.offs]

            argv.append(text)

        return s_ast.Const(tuple(argv))
1054963
def extract_dms (archive, compression, cmd, verbosity, interactive, outdir):
    """"""
    check_archive_ext(archive)
    cmdlist = [cmd, '-d', outdir]
    if verbosity > 1:
        cmdlist.append('-v')
    cmdlist.extend(['u', archive])
    return cmdlist
109377
def lrun(command, *args, **kwargs):
    ''''''
    return run('cd {0} && {1}'.format(ROOT, command), *args, **kwargs)
1004249
def shell(state, host, commands, chdir=None):
    '''
    
    '''

    # Ensure we have a list
    if isinstance(commands, six.string_types):
        commands = [commands]

    for command in commands:
        if chdir:
            yield 'cd {0} && ({1})'.format(chdir, command)
        else:
            yield command
2296
def cli(env):
    """"""
    filtered_vars = dict([(k, v)
                          for k, v in env.vars.items()
                          if not k.startswith('_')])
    env.fout(formatting.iter_to_table(filtered_vars))
1157298
def mv(hdfs_src, hdfs_dst):
    
    cmd = "hadoop fs -mv %s %s" % (hdfs_src, hdfs_dst)
    rcode, stdout, stderr = _checked_hadoop_fs_command(cmd)