Generates a summary report for all dataset in PMLB
    :param local_cache_dir: str (required)
        The directory on your local machine to store the data files.
185123
def generate_pmlb_summary(local_cache_dir=None):
    
    report_filename = open(os.path.join(local_cache_dir, 'report.csv'), 'wt')
    assert (local_cache_dir!=None)
    try:
        writer = csv.writer(report_filename, delimiter='\t')
        writer.writerow(['Dataset','#instances','#features','#binary_features','#integer_features','#float_features',\
                     'Endpoint_type','#classes','Imbalance_metric'])

        for dataset in dataset_names:
            df=fetch_data(dataset)
            print( "Dataset:", dataset)
            assert 'class' in df.columns, "no class column"
            #removing class column
            print( "SIZE: "+ str(len(df.axes[0]))+ " x " + str(len(df.axes[1])-1))
            feat=count_features_type(df.ix[:, df.columns != 'class'])
            endpoint=determine_endpoint_type(df.ix[:, df.columns == 'class'])
            mse=imbalance_metrics(df['class'].tolist())
            #writer.writerow([file,str(len(df.axes[0])),str(len(df.axes[1])-1),feat[0],feat[1],feat[2],endpoint,mse[0],mse[1],mse[2]])
            writer.writerow([dataset,str(len(df.axes[0])),str(len(df.axes[1])-1),feat[0],feat[1],feat[2],endpoint,int(mse[0]),mse[1]])
    finally:
        report_filename.close()
1222492
def pmlr(volumes='all', data_set='pmlr'):
    """"""
    if not data_available(data_set):
        download_data(data_set)
        
    proceedings_file = open(os.path.join(data_path, data_set, 'proceedings.yaml'), 'r')
    import yaml
    proceedings = yaml.load(proceedings_file)
    
    # Create a new resources entry for downloading contents of proceedings.
    data_name_full = 'pmlr_volumes'
    data_resources[data_name_full] = data_resources[data_set].copy()
    data_resources[data_name_full]['files'] = []
    data_resources[data_name_full]['dirs'] = []
    data_resources[data_name_full]['urls'] = []
    for entry in proceedings:
        if volumes=='all' or entry['volume'] in volumes:
            file = entry['yaml'].split('/')[-1]
            dir = 'v' + str(entry['volume'])
            data_resources[data_name_full]['files'].append([file])
            data_resources[data_name_full]['dirs'].append([dir])
            data_resources[data_name_full]['urls'].append(data_resources[data_set]['urls'][0])
    Y = []
    # Download the volume data
    if not data_available(data_name_full):
        download_data(data_name_full)
    for entry in reversed(proceedings):
        volume =  entry['volume']
        if volumes == 'all' or volume in volumes:
            file = entry['yaml'].split('/')[-1]
            volume_file = open(os.path.join(
                data_path, data_name_full,
                'v'+str(volume), file
                ), 'r')
            Y+=yaml.load(volume_file)
    if pandas_available:
        Y = pd.DataFrame(Y)
        Y['published'] = pd.to_datetime(Y['published'])
        #Y.columns.values[4] = json_object('authors')
        #Y.columns.values[7] = json_object('editors')
        Y['issued'] = Y['issued'].apply(lambda x: np.datetime64(datetime.datetime(*x['date-parts'])))
        Y['author'] = Y['author'].apply(lambda x: [str(author['given']) + ' ' + str(author['family']) for author in x])
        Y['editor'] = Y['editor'].apply(lambda x: [str(editor['given']) + ' ' + str(editor['family']) for editor in x])
        columns = list(Y.columns)
        columns[14] = datetime64_('published')
        columns[11] = datetime64_('issued')
        Y.columns = columns
        
    return data_details_return({'Y' : Y, 'info' : 'Data is a pandas data frame containing each paper, its abstract, authors, volumes and venue.'}, data_set)
620267
def output_directory(self) -> str:
        

        return os.path.join(self.results_path, 'reports', self.uuid, 'latest')
910034
def file_local_list(self, saltenv='base'):
        '''
        
        '''
        filesdest = os.path.join(self.opts['cachedir'], 'files', saltenv)
        localfilesdest = os.path.join(self.opts['cachedir'], 'localfiles')

        fdest = self._file_local_list(filesdest)
        ldest = self._file_local_list(localfilesdest)
        return sorted(fdest.union(ldest))
217173
def download_data():
    
    with urlopen(_retrieve_download_url()) as f:
        with open(os.path.join(CACHE_FOLDER, CACHE_ZIP), "wb") as local_file:
            local_file.write(f.read())
698377
async def add_local_charm_dir(self, charm_dir, series):
        
        fh = tempfile.NamedTemporaryFile()
        CharmArchiveGenerator(charm_dir).make_archive(fh.name)
        with fh:
            func = partial(
                self.add_local_charm, fh, series, os.stat(fh.name).st_size)
            charm_url = await self._connector.loop.run_in_executor(None, func)

        log.debug('Uploaded local charm: %s -> %s', charm_dir, charm_url)
        return charm_url
1209811
def load_data_split(proc_data_dir):
    
    ds_train = Dataset.load(path.join(proc_data_dir, 'train.bin'))
    ds_val = Dataset.load(path.join(proc_data_dir, 'val.bin'))
    ds_test = Dataset.load(path.join(proc_data_dir, 'test.bin'))
    return ds_train, ds_val, ds_test
404207
def _prepare_babi_data(tmp_dir, data_dir):
  
  if not tf.gfile.Exists(data_dir):
    tf.gfile.MakeDirs(data_dir)

  file_path = os.path.join(tmp_dir, _TAR)
  headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) "
                           "AppleWebKit/537.36 (KHTML, like Gecko) "
                           "Chrome/63.0.3239.132 Safari/537.36"}
  resp = requests.get(_URL, headers=headers)
  with open(file_path, "wb") as f:
    f.write(resp.content)

  tar = tarfile.open(file_path)
  tar.extractall(tmp_dir)
  tar.close()

  return tmp_dir
3826
def do_build_reports(directory):
    
    for cwd, dirs, files in os.walk(directory):
        for f in sorted(files):
            if f in ('job.ini', 'job_h.ini', 'job_haz.ini', 'job_hazard.ini'):
                job_ini = os.path.join(cwd, f)
                logging.info(job_ini)
                try:
                    reportwriter.build_report(job_ini, cwd)
                except Exception as e:
                    logging.error(str(e))
683209
def reports(self):
        
        printtime('Finding reports', self.start)
        # Initialise a list of runs and a list of all the metadata objects
        runs = list()
        samples = list()
        # Glob all the assembly folders into a list
        folders = glob(os.path.join(self.path, '*'.format(self.path)))
        # Iterate through all the folders
        for folder in folders:
            if os.path.isdir(folder):
                # Ignore any previously processed aggregate reports folder
                if 'reports' not in folder:
                    # Treat the external labs differently - instead of having all the assemblies in the path, each lab
                    # has its own subfolder with assemblies
                    if self.external:
                        subfolder = glob(os.path.join(folder, '*'))
                        for subsubfolder in subfolder:
                            if os.path.isdir(subfolder):
                                runs.append(subsubfolder)
                    else:
                        runs.append(folder)
        # Create metadata objects for each assembly run
        for assembly in sorted(runs):
            # Initialise the Metadata object
            metadata = MetadataObject()
            metadata.name = assembly.split('/')[-2] if assembly.split('/')[-3] == 'WGSspades' else '{}-{}'\
                .format(assembly.split('/')[-3], assembly.split('/')[-2])
            # Strip of any underscores and anything after an underscore
            metadata.name = metadata.name.split('_')[0]
            # Initialise Genobjects to store nested dictionaries of metadata in each metadata object
            metadata.general = GenObject()
            # Populate the GenObject
            metadata.general.path = assembly
            metadata.general.reportpath = os.path.join(str(assembly), 'reports')
            metadata.general.reports = glob(os.path.join(metadata.general.reportpath, '*.csv'))
            metadata.general.reportnames = [os.path.basename(x) for x in metadata.general.reports]
            # Add all the names of the reports to the set of reports
            for reportname in metadata.general.reportnames:
                self.reportset.add(reportname)
            # Add the metadata to list of metadata objects
            samples.append(metadata)
        # Return the list of metadata objects
        return samples
1016947
def generate_reports(subject_list, output_dir, work_dir, run_uuid, config=None,
                     packagename=None):
    
    reports_dir = str(Path(work_dir) / 'reportlets')
    report_errors = [
        run_reports(reports_dir, output_dir, subject_label, run_uuid,
                    config, packagename=packagename)
        for subject_label in subject_list
    ]

    errno = sum(report_errors)
    if errno:
        import logging
        logger = logging.getLogger('cli')
        error_list = ', '.join('%s (%d)' % (subid, err)
                               for subid, err in zip(subject_list, report_errors) if err)
        logger.error(
            'Preprocessing did not finish successfully. Errors occurred while processing '
            'data from participants: %s. Check the HTML reports for details.', error_list)
    return errno
167976
def _write_index_file(data_dir):
    
    cached_words = [
        w
        for w in _get_words(data_dir)
        if data_dir.joinpath("translations/{}.html".format(w)).is_file()
    ]

    content_str = _create_index_content(cached_words)
    html_string = HTML_TEMPLATE.replace("{% word %}", "Index")
    html_string = html_string.replace("{% content %}", content_str)

    filename = data_dir.joinpath("index.html")
    save_file(filename, html_string, mk_parents=True)
391748
def _generate_report_all(self):
        ''''''
        assert self.workbook is not None
        count = 0

        # Do all subfolders
        for sid in self.folders.subfolders(self.folder_id, self.user):
            count += 1
            self._generate_for_subfolder(sid)

        if count == 0:
            print("I: empty workbook created: no subfolders found")
1362575
def create_results_dirs(base_path):
    

    if not os.path.exists(base_path):
        print("Creating directory {} for results data.".format(base_path))
        os.mkdir(base_path)
    if not os.path.exists(os.path.join(base_path, 'results')):
        os.mkdir(os.path.join(base_path, 'results'))
    if not os.path.exists(os.path.join(base_path, 'plots')):
        os.mkdir(os.path.join(base_path, 'plots'))
    if not os.path.exists(os.path.join(base_path, 'info')):
        os.mkdir(os.path.join(base_path, 'info'))
    if not os.path.exists(os.path.join(base_path, 'log')):
        os.mkdir(os.path.join(base_path, 'log'))
936843
def _report(data, reference):
    
    seqcluster = op.join(get_bcbio_bin(), "seqcluster")
    work_dir = dd.get_work_dir(data)
    out_dir = safe_makedir(os.path.join(work_dir, "seqcluster", "report"))
    out_file = op.join(out_dir, "seqcluster.db")
    json = op.join(work_dir, "seqcluster", "cluster", "seqcluster.json")
    cmd = ("{seqcluster} report -o {out_dir} -r {reference} -j {json}")
    if not file_exists(out_file):
        do.run(cmd.format(**locals()), "Run report on clusters")
    return out_file
346061
def create_directories(self, create_project_dir=True):
        
        return task.create_directories(self.datadir, self.sitedir,
            self.target if create_project_dir else None)
431424
def populate_local_cache(self):
        
        fw_dict = self.get_all_fw_db()
        for fw_id in fw_dict:
            LOG.info("Populating cache for FW %s", fw_id)
            fw_data = fw_dict[fw_id]
            self.populate_local_cache_tenant(fw_id, fw_data)