Invoked the rados df command and return output to user
424852
def rados_df(self,
                 host_list=None,
                 remote_user=None,
                 remote_pass=None):
        '''
        
        '''
        result, failed_hosts = self.runner.ansible_perform_operation(
            host_list=host_list,
            remote_user=remote_user,
            remote_pass=remote_pass,
            module="command",
            module_args="rados df")

        parsed_result = self.rados_parse_df(result)

        return parsed_result
338723
def vcf_to_df(canvasvcfs, exonbed, cpus):
    
    df = pd.DataFrame()
    p = Pool(processes=cpus)
    results = []
    args = [(x, exonbed, i) for (i, x) in enumerate(canvasvcfs)]
    r = p.map_async(vcf_to_df_worker, args,
                    callback=results.append)
    r.wait()

    for res in results:
        df = df.append(res, ignore_index=True)
    return df
319162
def main(**kwargs):
    
    draw_lines()
    draw_histogram()
    draw_bar_chart()
    destination = "-r /report"
    if use_html:
        generate_html()
        command = "dx-build-report-html {h} {d}".format(h=html_filename, d=destination)
    else:
        command = "dx-build-report-html {l} {b} {h} {d}".format(l=lines_filename, b=bars_filename, h=histogram_filename, d=destination)
    sub_output = json.loads(subprocess.check_output(command, shell=True))
    output = {}
    output["report"] = dxpy.dxlink(sub_output["recordId"])
    return output
960227
def rock(self):
        """"""
        if not self.argv:
            self.arg.view()
        while(self.argv):
            arg = self.argv.popleft()
            if arg == "-h" or arg == "--help":
                print(
                    """Usage: td [-h (--help)] [-v (--version)] [command], where [command] is one of:\n\nv (view)\tChanges the way next output will look like. See [td v -h].\nm (modify)\tApplies one time changes to the database. See [td m -h].\no (options)\tSets persistent options, applied on every next execution. See [td o -h].\na (add)\t\tAdds new item. See [td a -h].\ne (edit)\tEdits existing item. See [td e -h].\nr (rm)\t\tRemoves existing item. See [td r -h].\nd (done)\tMarks items as done. See [td d -h].\nD (undone)\tMarks items as not done. See [td D -h].\n\nAdditional options:\n  -h (--help)\tShows this screen.\n  -v (--version)Shows version number."""
                )
            elif arg == "-v" or arg == "--version":
                print("td :: {}".format(__version__))
            elif arg == "v" or arg == "view":
                self._part("view", self.arg.view, {
                    "--no-color": ("nocolor", False),
                    "-s": ("sort", True), "--sort": ("sort", True),
                    "-p": ("purge", False), "--purge": ("purge", False),
                    "-d": ("done", True), "--done": ("done", True),
                    "-D": ("undone", True), "--undone": ("undone", True)
                },
                    """Usage: td v [-h (--help)] [command(s)], where [command(s)] are any of:\n\n-s (--sort) <pattern>\tSorts the output using <pattern>.\n-p (--purge)\t\tHides items marked as done.\n-d (--done) <pattern>\tDisplays items matching <pattern> as done.\n-D (--undone) <pattern>\tDisplays items matching <pattern> as not done.\n--no-color\t\tDo not add color codes to the output.\n\nAdditional options:\n  -h (--help)\t\tShows this screen."""
                )
            elif arg == "m" or arg == "modify":
                self._part("modify", self.arg.modify, {
                    "-s": ("sort", True), "--sort": ("sort", True),
                    "-p": ("purge", False), "--purge": ("purge", False),
                    "-d": ("done", True), "--done": ("done", True),
                    "-D": ("undone", True), "--undone": ("undone", True)
                },
                    """Usage: td m [-h (--help)] [command(s)], where [command(s)] are any of:\n\n-s (--sort) <pattern>\tSorts database using <pattern>.\n-p (--purge)\t\tRemoves items marked as done.\n-d (--done) <pattern>\tMarks items matching <pattern> as done.\n-D (--undone) <pattern>\tMarks items matching <pattern> as not done.\n\nAdditional options:\n  -h (--help)\t\tShows this screen."""
                )
            elif arg == "a" or arg == "add":
                args = dict()
                if self.argv and self.arg.model.exists(self.argv[0]):
                    args["parent"] = self.argv.popleft()
                self._part("add", self.arg.add, {
                    "-n": ("name", True), "--name": ("name", True),
                    "-p": ("priority", True), "--priority": ("priority", True),
                    "-c": ("comment", True), "--comment": ("comment", True)
                },
                    """Usage: td a [-h (--help)] [parent] [command(s)], where [command(s)] are any of:\n\n-n (--name) <text>\t\tSets item's name.\n-p (--priority) <no|name>\tSets item's priority.\n-c (--comment) <text>\t\tSets item's comment.\n\nIf [parent] index is specified, new item will become it's child.\nIf any of the arguments is omitted, this command will launch an interactive session letting the user supply the rest of them.\n\nAdditional options:\n  -h (--help)\t\t\tShows this screen.""",
                    **args
                )
            elif arg == "e" or arg == "edit":
                if not self.argv:
                    raise NotEnoughArgumentsError("edit")
                args = dict()
                if self.argv[0] not in ["-h", "--help"]:
                    args["index"] = self.argv.popleft()
                self._part("edit", self.arg.edit, {
                    "--parent": ("parent", True),
                    "-n": ("name", True), "--name": ("name", True),
                    "-p": ("priority", True), "--priority": ("priority", True),
                    "-c": ("comment", True), "--comment": ("comment", True)
                },
                    """Usage: td e [-h (--help)] <index> [command(s)], where [command(s)] are any of:\n\n--parent <index>\t\tChanges item's parent.\n-n (--name) <text>\t\tChanges item's name.\n-p (--priority) <no|name>\tChanges item's priority.\n-c (--comment) <text>\t\tChanges item's comment.\n\nIndex argument is required and has to point at an existing item.\nIf any of the arguments is omitted, it will launch an interactive session letting the user supply the rest of them.\n\nAdditions options:\n  -h (--help)\t\t\tShows this screen.""",
                    **args
                )
            elif arg == "r" or arg == "rm":
                args = dict()
                if not self.argv:
                    raise NotEnoughArgumentsError("rm")
                elif self.argv[0] not in ["-h", "--help"]:
                    args["index"] = self.argv.popleft()
                self._part("rm", self.arg.rm, {
                },
                    """Usage: td r [-h (--help)] <index>\n\nIndex argument is required and has to point at an existing item.\n\nAdditions options:\n  -h (--help)\tShows this screen.""",
                    **args
                )
            elif arg == "d" or arg == "done":
                args = dict()
                if not self.argv:
                    raise NotEnoughArgumentsError("done")
                elif self.argv[0] not in ["-h", "--help"]:
                    args["index"] = self.argv.popleft()
                self._part("done", self.arg.done, {
                },
                    """Usage: td d [-h (--help)] <index>\n\nIndex argument is required and has to point at an existing item.\n\nAdditional options:\n  -h (--help)\tShows this screen.""",
                    **args
                )
            elif arg == "D" or arg == "undone":
                args = dict()
                if not self.argv:
                    raise NotEnoughArgumentsError("undone")
                elif self.argv[0] not in ["-h", "--help"]:
                    args["index"] = self.argv.popleft()
                self._part("undone", self.arg.undone, {
                },
                    """Usage: td D [-h (--help)] <index>\n\nIndex argument is required and has to point at an existing item.\n\nAdditional options:\n  -h (--help)\tShows this screen.""",
                    **args
                )
            elif arg == "o" or arg == "options":
                self._part("options", self.arg.options, {
                    "-g": ("glob", False), "--global": ("glob", False),
                    "-s": ("sort", True), "--sort": ("sort", True),
                    "-p": ("purge", False), "--purge": ("purge", False),
                    "-d": ("done", True), "--done": ("done", True),
                    "-D": ("undone", True), "--undone": ("undone", True)
                },
                    """Usage: td o [-h (--help)] [command(s)], where [command(s)] are any of:\n\n-g (--global)\t\tApply specified options to all ToDo lists (store in ~/.tdrc).\n-s (--sort) <pattern>\tAlways sorts using <pattern>.\n-p (--purge)\t\tAlways removes items markedas done.\n-d (--done) <pattern>\tAlways marks items maching <pattern> as done.\n-D (--undone) <pattern>\tAlways marks items maching <pattern> as not done.\n\nAdditional options:\n  -h (--help)\t\tShows this screen."""
                )
            else:
                raise UnrecognizedCommandError("td", arg)
31970
def process(df, **kwargs):
    '''
    
    '''
    env, agents = split_df(df)
    return process_one(env, **kwargs), process_one(agents, **kwargs)
261033
def from_df(cls, path, df:DataFrame, dep_var:str, valid_idx:Collection[int], procs:OptTabTfms=None,
                cat_names:OptStrList=None, cont_names:OptStrList=None, classes:Collection=None, 
                test_df=None, bs:int=64, val_bs:int=None, num_workers:int=defaults.cpus, dl_tfms:Optional[Collection[Callable]]=None, 
                device:torch.device=None, collate_fn:Callable=data_collate, no_check:bool=False)->DataBunch:
        ""
        cat_names = ifnone(cat_names, []).copy()
        cont_names = ifnone(cont_names, list(set(df)-set(cat_names)-{dep_var}))
        procs = listify(procs)
        src = (TabularList.from_df(df, path=path, cat_names=cat_names, cont_names=cont_names, procs=procs)
                           .split_by_idx(valid_idx))
        src = src.label_from_df(cols=dep_var) if classes is None else src.label_from_df(cols=dep_var, classes=classes)
        if test_df is not None: src.add_test(TabularList.from_df(test_df, cat_names=cat_names, cont_names=cont_names,
                                                                 processor = src.train.x.processor))
        return src.databunch(path=path, bs=bs, val_bs=val_bs, num_workers=num_workers, device=device, 
                             collate_fn=collate_fn, no_check=no_check)
2373
def cli(ctx, **kwargs):
    
    if kwargs['add_sys_path']:
        sys.path.append(os.getcwd())

    logging.config.fileConfig(kwargs['logging_config'])

    # get db from env
    for db in ('taskdb', 'projectdb', 'resultdb'):
        if kwargs[db] is not None:
            continue
        if os.environ.get('MYSQL_NAME'):
            kwargs[db] = utils.Get(lambda db=db: connect_database(
                'sqlalchemy+mysql+%s://%s:%s/%s' % (
                    db, os.environ['MYSQL_PORT_3306_TCP_ADDR'],
                    os.environ['MYSQL_PORT_3306_TCP_PORT'], db)))
        elif os.environ.get('MONGODB_NAME'):
            kwargs[db] = utils.Get(lambda db=db: connect_database(
                'mongodb+%s://%s:%s/%s' % (
                    db, os.environ['MONGODB_PORT_27017_TCP_ADDR'],
                    os.environ['MONGODB_PORT_27017_TCP_PORT'], db)))
        elif ctx.invoked_subcommand == 'bench':
            if kwargs['data_path'] == './data':
                kwargs['data_path'] += '/bench'
                shutil.rmtree(kwargs['data_path'], ignore_errors=True)
                os.mkdir(kwargs['data_path'])
            if db in ('taskdb', 'resultdb'):
                kwargs[db] = utils.Get(lambda db=db: connect_database('sqlite+%s://' % (db)))
            elif db in ('projectdb', ):
                kwargs[db] = utils.Get(lambda db=db: connect_database('local+%s://%s' % (
                    db, os.path.join(os.path.dirname(__file__), 'libs/bench.py'))))
        else:
            if not os.path.exists(kwargs['data_path']):
                os.mkdir(kwargs['data_path'])
            kwargs[db] = utils.Get(lambda db=db: connect_database('sqlite+%s:///%s/%s.db' % (
                db, kwargs['data_path'], db[:-2])))
            kwargs['is_%s_default' % db] = True

    # create folder for counter.dump
    if not os.path.exists(kwargs['data_path']):
        os.mkdir(kwargs['data_path'])

    # message queue, compatible with old version
    if kwargs.get('message_queue'):
        pass
    elif kwargs.get('amqp_url'):
        kwargs['message_queue'] = kwargs['amqp_url']
    elif os.environ.get('RABBITMQ_NAME'):
        kwargs['message_queue'] = ("amqp://guest:guest@%(RABBITMQ_PORT_5672_TCP_ADDR)s"
                                   ":%(RABBITMQ_PORT_5672_TCP_PORT)s/%%2F" % os.environ)
    elif kwargs.get('beanstalk'):
        kwargs['message_queue'] = "beanstalk://%s/" % kwargs['beanstalk']

    for name in ('newtask_queue', 'status_queue', 'scheduler2fetcher',
                 'fetcher2processor', 'processor2result'):
        if kwargs.get('message_queue'):
            kwargs[name] = utils.Get(lambda name=name: connect_message_queue(
                name, kwargs.get('message_queue'), kwargs['queue_maxsize']))
        else:
            kwargs[name] = connect_message_queue(name, kwargs.get('message_queue'),
                                                 kwargs['queue_maxsize'])

    # phantomjs-proxy
    if kwargs.get('phantomjs_proxy'):
        pass
    elif os.environ.get('PHANTOMJS_NAME'):
        kwargs['phantomjs_proxy'] = os.environ['PHANTOMJS_PORT_25555_TCP'][len('tcp://'):]

    # puppeteer-proxy
    if kwargs.get('puppeteer_proxy'):
        pass
    elif os.environ.get('PUPPETEER_NAME'):
        kwargs['puppeteer_proxy'] = os.environ['PUPPETEER_PORT_22222_TCP'][len('tcp://'):]

    ctx.obj = utils.ObjectDict(ctx.obj or {})
    ctx.obj['instances'] = []
    ctx.obj.update(kwargs)

    if ctx.invoked_subcommand is None and not ctx.obj.get('testing_mode'):
        ctx.invoke(all)
    return ctx
903202
def output(results_dir, config, parent='../../'):
    
    start = time.time()
    print("Compiling results...")
    results_dir = os.path.abspath(results_dir)
    results = ReportResults(config['run_time'], config['results_ts_interval'])
    results.compile_results()
    print("Results compiled in {} seconds\n".format(time.time() - start))

    if results.total_transactions == 0:
        print("No results, cannot create report")
        return False

    print_infos(results)

    data = {
        'report': results,
        'run_time': config['run_time'],
        'ts_interval': config['results_ts_interval'],
        'turrets_config': results.turrets,
        'results': {"all": results.main_results, "timers": results.timers_results}
    }

    print("Generating graphs...")
    partial = time.time()
    generate_graphs(results.main_results, 'All_Transactions', results_dir)

    for key, value in results.timers_results.items():
        generate_graphs(value, key, results_dir)
    print("All graphs generated in {} seconds\n".format(time.time() - partial))

    write_template(data, results_dir, parent)
    print("Full report generated in {} seconds".format(time.time() - start))
    return True
576933
def output(ret, bar, **kwargs):  # pylint: disable=unused-argument
    '''
    
    '''
    if 'return_count' in ret:
        val = ret['return_count']
        # Avoid to fail if targets are behind a syndic. In this case actual return count will be
        # higher than targeted by MoM itself.
        # TODO: implement a way to get the proper target minions count and remove this workaround.
        # Details are in #44239.
        if val > bar.maxval:
            bar.maxval = val
        bar.update(val)
    return ''
407467
def run(args):
    

    with warnings.catch_warnings():
        warnings.simplefilter('ignore')

        query = prepareQuery(args.query_file.read())
        ds = Dataset()

        res_indices_prev = set()  # de-duplication
        res_indices = set()

        # create sub graphs
        for f in args.graphs:
            g = Graph(identifier=os.path.basename(f.name))
            g.parse(data=f.read(), format='n3')
            ds.add_graph(g)

        # create and query data graph
        for data in read_by_chunk(args.input_file, int(args.chunk_size)):
            g = Graph(identifier='data')
            g.parse(data=data, format=args.input_type)
            ds.add_graph(g)
            res = ds.query(query)

            dedup_res_graph = Graph()
            if len(res) != 0:
                for r in res:
                    tid = generate_index(r)
                    res_indices.add(tid)
                    if tid in res_indices_prev:  # duplicated
                        continue
                    dedup_res_graph.add(r)

                if len(dedup_res_graph) > 0:
                    ret = dedup_res_graph.serialize(format=args.output_type, encoding='utf-8')
                    args.output_file.write(ret)

            ds.remove_graph(g)
            res_indices_prev = res_indices
            res_indices = set()
446781
def call_output(cmd, stdin=None, encoding_errors="replace", **kwargs):
    """"""
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs)
    stdout, stderr, retcode = [], [], None
    while retcode is None:
        if stdin is not None:
            logger.log_prefix("<0 ", stdin.rstrip())
        raw_out, raw_err = p.communicate(stdin)
        stdin = None

        out = raw_out.decode(get_encoding(sys.stdout), encoding_errors) if raw_out else ""
        if out:
            logger.log_prefix("1> ", out.rstrip())
        stdout.append(out)

        err = raw_err.decode(get_encoding(sys.stderr), encoding_errors) if raw_err else ""
        if err:
            logger.log_prefix("2> ", err.rstrip())
        stderr.append(err)

        retcode = p.poll()
    return stdout, stderr, retcode
1743
def fromDataFrameRDD(cls, rdd, sql_ctx):
        """"""
        result = DataFrame(None, sql_ctx)
        return result.from_rdd_of_dataframes(rdd)
1306311
def __execute_cmd(command):
    '''
    
    '''
    cmd = __salt__['cmd.run_all']('racadm {0}'.format(command))

    if cmd['retcode'] != 0:
        log.warning('racadm return an exit code \'%s\'.', cmd['retcode'])
        return False

    return True
496616
def main():
    
    input_tif = "../tests/data/Jamaica_dem.tif"
    output_tif = "../tests/data/tmp_results/log_dem.tif"
    rst = RasterUtilClass.read_raster(input_tif)
    # raster data (with noDataValue as numpy.nan) as numpy array
    rst_valid = rst.validValues
    output_data = np.log(rst_valid)
    # write output raster
    RasterUtilClass.write_gtiff_file(output_tif, rst.nRows, rst.nCols, output_data, rst.geotrans,
                                     rst.srs, rst.noDataValue, rst.dataType)
693642
def output_results(results, mvdelim = '\n', output = sys.stdout):
    """"""

    # We collect all the unique field names, as well as 
    # convert all multivalue keys to the right form
    fields = set()
    for result in results:    
        for key in result.keys():
            if(isinstance(result[key], list)):
                result['__mv_' + key] = encode_mv(result[key])
                result[key] = mvdelim.join(result[key])
        fields.update(list(result.keys()))

    # convert the fields into a list and create a CSV writer
    # to output to stdout
    fields = sorted(list(fields))

    writer = csv.DictWriter(output, fields)

    # Write out the fields, and then the actual results
    writer.writerow(dict(list(zip(fields, fields))))
    writer.writerows(results)
661281
def collect_cmd(self, args, ret):
        """"""
        from dvc.command.daemon import CmdDaemonAnalytics

        assert isinstance(ret, int) or ret is None

        if ret is not None:
            self.info[self.PARAM_CMD_RETURN_CODE] = ret

        if args is not None and hasattr(args, "func"):
            assert args.func != CmdDaemonAnalytics
            self.info[self.PARAM_CMD_CLASS] = args.func.__name__
21027
def write_command(self, command: Command):
        '''
        '''
        _logger.debug('Write command.')
        data = command.to_bytes()
        yield from self._connection.write(data)
        self._data_event_dispatcher.notify_write(data)