Generate all reports from original dataframe

    :param dic data: dict containing raw and compiled results dataframes
    :param str name: name for prefixing graphs output
    :param str results_dir: results output directory
968030
def generate_graphs(data, name, results_dir):
    
    graphs.resp_graph_raw(data['raw'], name + '_response_times.svg', results_dir)
    graphs.resp_graph(data['compiled'], name + '_response_times_intervals.svg', results_dir)
    graphs.tp_graph(data['compiled'], name + '_throughput.svg', results_dir)
462121
def get_result_files(self, result):
        
        if isinstance(result, dict):
            result_id = result['meta']['id']
        else:  # Should already be a string containing the id
            result_id = result

        result_data_dir = os.path.join(self.get_data_dir(), result_id)

        filenames = next(os.walk(result_data_dir))[2]

        filename_path_pairs = [
            (f, os.path.join(self.get_data_dir(), result_id, f))
            for f in filenames]

        return {k: v for k, v in filename_path_pairs}
242282
def _load_from_results_dir(self, compile_classpath, vts_results_dir,
                             coursier_cache_path, invalidation_check, pants_jar_path_base):
    
    result_file_path = os.path.join(vts_results_dir, self.RESULT_FILENAME)
    if not os.path.exists(result_file_path):
      return

    with open(result_file_path, 'r') as f:
      results = json.load(f)
      for conf, result_list in results.items():
        for result in result_list:
          try:
            self._load_json_result(conf,
                                   compile_classpath,
                                   coursier_cache_path,
                                   invalidation_check,
                                   pants_jar_path_base,
                                   result,
                                   self._override_classifiers_for_conf(conf))
          except CoursierResultNotFound:
            return False

    return True
1181543
def results_to_csv(query_name, **kwargs):
    
    query = get_result_set(query_name, **kwargs)
    result = query.result
    columns = list(result[0].keys())
    data = [tuple(row.values()) for row in result]
    frame = tablib.Dataset()
    frame.headers = columns
    for row in data:
        frame.append(row)
    csvs = frame.export('csv')
    return csvs
331823
def results(self):
        ""
        if self._results is None:
            results = self._json_struct['results']
            def result_iterator():
                for result in results:
                    datatype = None
                    conversion = None
                    for param in self.parent.parameters:
                        if param['name'] == result['paramName']:
                            datatype = param['datatype']
                    if datatype is None:
                        conversion = str
                    else:
                        conversion = datatype.fromJson
                    dt = result['paramName']
                    val = conversion(result['value'])
                    yield (dt, val)
            self._results = dict(res for res in result_iterator())
        return self._results
792239
def collect_results(working_folder):
    
    results = {} 
    working_folder = Path(working_folder)
    output_folders = working_folder/'output_folders'

    for rec_name in os.listdir(output_folders):
        if not os.path.isdir(output_folders / rec_name):
            continue
        # print(rec_name)
        results[rec_name] = {}
        for sorter_name in os.listdir(output_folders / rec_name):
            # print('  ', sorter_name)
            output_folder = output_folders / rec_name / sorter_name
            #~ print(output_folder)
            if not os.path.isdir(output_folder):
                continue
            SorterClass = sorter_dict[sorter_name]
            results[rec_name][sorter_name] = SorterClass.get_result_from_folder(output_folder)

    return results
103337
def get_all_datasets(self):
        
        success = True

        for dataset in tqdm(self.datasets):
            individual_success = self.get_dataset(dataset)
            if not individual_success:
                success = False

        return success
388550
def merge_batch_results(self, result_ids):
        
        outfile = open(self.output().path, 'w')

        if self.content_type.lower() == 'csv':
            for i, result_id in enumerate(result_ids):
                with open("%s.%d" % (self.output().path, i), 'r') as f:
                    header = f.readline()
                    if i == 0:
                        outfile.write(header)
                    for line in f:
                        outfile.write(line)
        else:
            raise Exception("Batch result merging not implemented for %s" % self.content_type)

        outfile.close()
1114420
def generate_flattened_dict(headers, results):
    ''''''
    for name, values in results.items():
        row = []
        for header in headers:
            if header == 'name':
                row.append(name)
            elif ':' in header:
                neurite_type, metric = header.split(':')
                row.append(values[neurite_type][metric])
            else:
                row.append(values[header])
        yield row
185123
def generate_pmlb_summary(local_cache_dir=None):
    
    report_filename = open(os.path.join(local_cache_dir, 'report.csv'), 'wt')
    assert (local_cache_dir!=None)
    try:
        writer = csv.writer(report_filename, delimiter='\t')
        writer.writerow(['Dataset','#instances','#features','#binary_features','#integer_features','#float_features',\
                     'Endpoint_type','#classes','Imbalance_metric'])

        for dataset in dataset_names:
            df=fetch_data(dataset)
            print( "Dataset:", dataset)
            assert 'class' in df.columns, "no class column"
            #removing class column
            print( "SIZE: "+ str(len(df.axes[0]))+ " x " + str(len(df.axes[1])-1))
            feat=count_features_type(df.ix[:, df.columns != 'class'])
            endpoint=determine_endpoint_type(df.ix[:, df.columns == 'class'])
            mse=imbalance_metrics(df['class'].tolist())
            #writer.writerow([file,str(len(df.axes[0])),str(len(df.axes[1])-1),feat[0],feat[1],feat[2],endpoint,mse[0],mse[1],mse[2]])
            writer.writerow([dataset,str(len(df.axes[0])),str(len(df.axes[1])-1),feat[0],feat[1],feat[2],endpoint,int(mse[0]),mse[1]])
    finally:
        report_filename.close()
191518
def _create_variables_dic(self):
        
        self.name_to_variable = {}
        for variable in self.space_expanded:
            self.name_to_variable[variable.name] = variable
1324354
def generate_dir_rst(dir, fhindex, example_dir, root_dir, plot_gallery):
    
    if not dir == '.':
        target_dir = os.path.join(root_dir, dir)
        src_dir = os.path.join(example_dir, dir)
    else:
        target_dir = root_dir
        src_dir = example_dir
    if not os.path.exists(os.path.join(src_dir, 'README.txt')):
        print('Example directory %s does not have a README.txt file' % src_dir)
        print('Skipping this directory')
        print(80 * '_')
        return
    fhindex.write("""


%s


""" % file(os.path.join(src_dir, 'README.txt')).read())
    if not os.path.exists(target_dir):
        os.makedirs(target_dir)

    def sort_key(a):
        # put last elements without a plot
        if not a.startswith('plot') and a.endswith('.py'):
            return 'zz' + a
        return a

    fhindex.write("""

.. toctree::

""")
    for fname in sorted(os.listdir(src_dir), key=sort_key):
        if fname.endswith('py'):
            generate_file_rst(fname, target_dir, src_dir, plot_gallery)
            #thumb = os.path.join(dir, 'images', 'thumb', fname[:-3] + '.png')
            link_name = os.path.join(dir, fname).replace(os.path.sep, '_')
            #fhindex.write('.. figure:: %s\n' % thumb)
            #if link_name.startswith('._'):
            #    link_name = link_name[2:]
            #if dir != '.':
            #    fhindex.write('   :target: ./%s/%s.html\n\n' % (dir,
            #                                                   fname[:-3]))
            #else:
            #    fhindex.write('   :target: ./%s.html\n\n' % link_name[:-3])
            fhindex.write("""
   %s/%s
""" % (dir, fname[:-3]))
    fhindex.write("""

.. raw:: html

    <div style="clear: both"></div>
    """)
606227
def _output_results(
            self):
        
        self.log.info('starting the ``_output_results`` method')

        content = ""
        maxNameLen = 0

        for r in self.results:
            if maxNameLen < len(r["ned_name"]):
                maxNameLen = len(r["ned_name"])

        if len(self.results) == 0:
            content += "No resuls found"
        else:
            thisHeader = "| "
            thisLine = "| "
            for head in self.headers:
                if head == "ned_name":
                    s = maxNameLen
                else:
                    s = self.resultSpacing
                thisHeader += str(head).ljust(s,
                                              ' ') + " | "
                thisLine += ":".ljust(s,
                                      '-') + " | "
            content += thisHeader
            content += "\n" + thisLine
            for r in self.results:
                thisRow = "| "
                for head in self.headers:
                    if head == "ned_name":
                        s = maxNameLen
                    else:
                        s = self.resultSpacing
                    thisRow += str(r[head]).ljust(s,
                                                  ' ') + " | "
                content += "\n" + thisRow

        if self.quiet == False:
            print content
        if self.outputFilePath:
            import codecs
            writeFile = codecs.open(
                self.outputFilePath, encoding='utf-8', mode='w')
            writeFile.write(content)
            writeFile.close()

        self.log.info('completed the ``_output_results`` method')
        return None
137285
def convert_results(data, **kwargs):
    
    if kwargs.get("multiprocessing", False):
        manager = SharedManager()
        manager.register("BaseRdfDataType", BaseRdfDataType)
        manager.register("Uri", Uri)

        data_l = len(data)
        group_size = data_l // pool_size
        if data_l % pool_size:
            group_size += 1
        split_data = [data[i:i + group_size]
                      for i in range(0, data_l, group_size)]
        output = manager.Queue()
        # output = manager.list()
        # output_data = POOL.map(convert_row, split_data)
        workers = [mp.Process(target=convert_batch, args=(item, output,))
                   for item in split_data]
        for worker in workers:
            # worker.Daemon = True
            worker.start()
        results = []
        while True:
            running = any(p.is_alive() for p in workers)
            while not output.empty():
               results += output.get()
            if not running:
                break
        print("Finished - workers not stoped")
        for worker in workers:
            worker.join()
        # pdb.set_trace()
        # return output
        for i in range(output.qsize()):
            results += output.get()
        return results
    else:
        return [{key:pyrdf(value) for key, value in row.items()}
                for row in data]
1129051
def directory_generator(dirname, trim=0):
    

    def gather(collect, dirname, fnames):
        for fname in fnames:
            df = join(dirname, fname)
            if not isdir(df):
                collect.append(df)

    collect = list()
    walk(dirname, gather, collect)
    for fname in collect:
        yield fname[trim:], file_chunk(fname)
1016947
def generate_reports(subject_list, output_dir, work_dir, run_uuid, config=None,
                     packagename=None):
    
    reports_dir = str(Path(work_dir) / 'reportlets')
    report_errors = [
        run_reports(reports_dir, output_dir, subject_label, run_uuid,
                    config, packagename=packagename)
        for subject_label in subject_list
    ]

    errno = sum(report_errors)
    if errno:
        import logging
        logger = logging.getLogger('cli')
        error_list = ', '.join('%s (%d)' % (subid, err)
                               for subid, err in zip(subject_list, report_errors) if err)
        logger.error(
            'Preprocessing did not finish successfully. Errors occurred while processing '
            'data from participants: %s. Check the HTML reports for details.', error_list)
    return errno
669054
def print_rendered_results(results_dict):
    
    class _HubComponentEncoder(json.JSONEncoder):
        def default(self, o):
            if isinstance(o, base.HubComponent):
                return repr(o)
            return json.JSONEncoder.default(self, o)
    formatted = json.dumps(results_dict, indent=4, cls=_HubComponentEncoder)
    # the returned string contains lines with trailing spaces, which causes
    # doctests to fail.  So fix that here.
    for s in formatted.splitlines():
        print(s.rstrip())