Create download jobs for all file formats from a summary file entry.
553002
def create_downloadjob(entry, domain, config):
    """"""
    logging.info('Checking record %r', entry['assembly_accession'])
    full_output_dir = create_dir(entry, config.section, domain, config.output)

    symlink_path = None
    if config.human_readable:
        symlink_path = create_readable_dir(entry, config.section, domain, config.output)

    checksums = grab_checksums_file(entry)

    # TODO: Only write this when the checksums file changed
    with open(os.path.join(full_output_dir, 'MD5SUMS'), 'w') as handle:
        handle.write(checksums)

    parsed_checksums = parse_checksums(checksums)

    download_jobs = []
    for fmt in config.file_format:
        try:
            if has_file_changed(full_output_dir, parsed_checksums, fmt):
                download_jobs.append(
                    download_file_job(entry, full_output_dir, parsed_checksums, fmt, symlink_path))
            elif need_to_create_symlink(full_output_dir, parsed_checksums, fmt, symlink_path):
                download_jobs.append(
                    create_symlink_job(full_output_dir, parsed_checksums, fmt, symlink_path))
        except ValueError as err:
            logging.error(err)

    return download_jobs
502039
def generate(self):
        """"""
        for root, _, files in os.walk(self.source_dir):
            for fname in files:
                source_fpath = os.path.join(root, fname)
                self.generate_api_for_source(source_fpath)
1362218
def download(ctx):
    """"""
    settings.add_cli_options(ctx.cli_options, settings.TransferAction.Download)
    ctx.initialize(settings.TransferAction.Download)
    specs = settings.create_download_specifications(
        ctx.cli_options, ctx.config)
    del ctx.cli_options
    for spec in specs:
        blobxfer.api.Downloader(
            ctx.general_options, ctx.credentials, spec
        ).start()
204354
def run(self):
        """"""
        self.batch = self.tcex.batch(self.args.tc_owner)

        # using tcex requests to get built-in features (e.g., proxy, logging, retries)
        request = self.tcex.request()

        with request.session as s:
            r = s.get(self.url)

            if r.ok:
                decoded_content = r.content.decode('utf-8').splitlines()

                reader = csv.reader(decoded_content, delimiter=',', quotechar='"')
                for row in reader:
                    # CSV headers
                    # Firstseen,MD5hash,Malware

                    # skip comments
                    if row[0].startswith('#'):
                        continue

                    # create batch entry
                    file_hash = self.batch.file(row[1], rating='4.0', confidence='100')
                    file_hash.tag(row[2])
                    occurrence = file_hash.occurrence()
                    occurrence.date = row[0]
                    self.batch.save(file_hash)  # optionally save object to disk
            else:
                self.tcex.exit(1, 'Failed to download CSV data.')

        # submit batch job
        batch_status = self.batch.submit_all()
        print(batch_status)

        self.exit_message = 'Downloaded data and create batch job.'
1123902
def create_job(self,
                   builder_job,    # type: Dict[Text, Any]
                   wf_job=None,    # type: Callable[[Dict[Text, Text], Callable[[Any, Any], Any], RuntimeContext], Generator[Any, None, None]]
                   is_output=False
                  ):  # type: (...) -> Dict
        #TODO customise the file
        """"""
        copied = copy.deepcopy(builder_job)
        relativised_input_objecttemp = {}  # type: Dict[Text, Any]
        self._relativise_files(copied)
        def jdefault(o):
            return dict(o)
        if is_output:
            rel_path = posixpath.join(_posix_path(WORKFLOW), "primary-output.json")
        else:
            rel_path = posixpath.join(_posix_path(WORKFLOW), "primary-job.json")
        j = json_dumps(copied, indent=4, ensure_ascii=False, default=jdefault)
        with self.write_bag_file(rel_path) as file_path:
            file_path.write(j + u"\n")
        _logger.debug(u"[provenance] Generated customised job file: %s",
                      rel_path)
        # Generate dictionary with keys as workflow level input IDs and values
        # as
        # 1) for files the relativised location containing hash
        # 2) for other attributes, the actual value.
        relativised_input_objecttemp = {}
        for key, value in copied.items():
            if isinstance(value, MutableMapping):
                if value.get("class") in ("File", "Directory"):
                    relativised_input_objecttemp[key] = value
            else:
                relativised_input_objecttemp[key] = value
        self.relativised_input_object.update(
            {k: v for k, v in relativised_input_objecttemp.items() if v})
        return self.relativised_input_object
754353
def export_comps(request):
    
    in_memory = BytesIO()
    zip = ZipFile(in_memory, "a")

    comps = settings.COMPS_DIR
    static = settings.STATIC_ROOT or ""
    context = RequestContext(request, {})
    context['debug'] = False

    # dump static resources
    # TODO: inspect each template and only pull in resources that are used
    for dirname, dirs, filenames in os.walk(static):
        for filename in filenames:
            full_path = os.path.join(dirname, filename)
            rel_path = os.path.relpath(full_path, static)
            content = open(full_path, 'rb').read()
            try:
                ext = os.path.splitext(filename)[1]
            except IndexError:
                pass
            if ext == '.css':
                # convert static refs to relative links
                dotted_rel = os.path.relpath(static, full_path)
                new_rel_path = '{0}{1}'.format(dotted_rel, '/static')
                content = content.replace(b'/static', bytes(new_rel_path, 'utf8'))
            path = os.path.join('static', rel_path)
            zip.writestr(path, content)

    for dirname, dirs, filenames in os.walk(comps):
        for filename in filenames:
            full_path = os.path.join(dirname, filename)
            rel_path = os.path.relpath(full_path, comps)
            template_path = os.path.join(comps.split('/')[-1], rel_path)
            html = render_to_string(template_path, context)
            # convert static refs to relative links
            depth = len(rel_path.split(os.sep)) - 1
            if depth == 0:
                dotted_rel = '.'
            else:
                dotted_rel = ''
                i = 0
                while i < depth:
                    dotted_rel += '../'
                    i += 1
            new_rel_path = '{0}{1}'.format(dotted_rel, '/static')
            html = html.replace('/static', new_rel_path)
            if PY2:
                html = unicode(html)
            zip.writestr(rel_path, html.encode('utf8'))

    for item in zip.filelist:
        item.create_system = 0
    zip.close()

    response = HttpResponse(content_type="application/zip")
    response["Content-Disposition"] = "attachment; filename=comps.zip"
    in_memory.seek(0)
    response.write(in_memory.read())

    return response
913297
def collect_jobs(dirs, runscript, overwrite=False, max_job_age=90):
    """"""

    jobs = []

    for dirname in sorted(dirs):

        o = dict(cfgfile=os.path.join(dirname, 'config.yaml'),
                 logfile=os.path.join(
                     dirname, os.path.splitext(runscript)[0] + '.log'),
                 runscript=os.path.join(dirname, runscript))

        if not os.path.isfile(o['cfgfile']):
            continue

        if not os.path.isfile(o['runscript']):
            continue

        if not os.path.isfile(o['logfile']):
            jobs.append(o)
            continue

        age = file_age_in_seconds(o['logfile']) / 60.
        job_status = check_log(o['logfile'])

        print(dirname, job_status, age)

        if job_status is False or overwrite:
            jobs.append(o)
        elif job_status == 'Exited':
            print("Job Exited. Resending command.")
            jobs.append(o)
        elif job_status == 'None' and age > max_job_age:
            print(
                "Job did not exit, but no activity on log file for > %.2f min. Resending command." % max_job_age)
            jobs.append(o)
        #        elif job_status is True:
        #            print("Job Completed. Resending command.")
        #            jobs.append(o)

    return jobs
801033
def download_shared_files(job, input_args):
    
    shared_files = ['unc.bed', 'hg19.transcripts.fa', 'composite_exons.bed', 'normalize.pl', 'rsem_ref.zip',
                    'ebwt.zip', 'chromosomes.zip']
    shared_ids = {}
    for f in shared_files:
        shared_ids[f] = job.addChildJobFn(download_from_url, input_args[f]).rv()
    if input_args['config'] or input_args['config_fastq']:
        job.addFollowOnJobFn(parse_config_file, shared_ids, input_args)
    else:
        sample_path = input_args['input']
        uuid = os.path.splitext(os.path.basename(sample_path))[0]
        sample = (uuid, sample_path)
        job.addFollowOnJobFn(download_sample, shared_ids, input_args, sample)
733852
protected function gatherHookDocumentationFiles($job_list) {
    $directory = \DrupalCodeBuilder\Factory::getEnvironment()->getHooksDirectory();

    // Fetch data about the files we need to download.
    $hook_files = $this->getHookFileUrls($directory);
    //print_r($hook_files);

    // Retrieve each file and store it in the hooks directory, overwriting what's currently there
    foreach ($hook_files as $file_name => $data) {
      $file_contents = drupal_http_request($data['url']);

      // TODO: replace with call to environment output.
      file_put_contents("$directory/$file_name", $destination . $file_contents->data);
    }

    return $hook_files;
  }
4559
def create_metadata_file(output_path, data_dir):
    '''
    '''

    # Relative path to all (BASIS).metadata.json files
    meta_filelist, table_filelist, _, _ = get_all_filelist(data_dir)

    metadata = {}
    for meta_file_relpath in meta_filelist:

        # Read in the metadata for a single basis set
        meta_file_path = os.path.join(data_dir, meta_file_relpath)
        bs_metadata = read_json_basis(meta_file_path)

        # Base of the filename for table basis sets
        # Basename is something like '6-31G.', including the last period
        base_relpath, meta_filename = os.path.split(meta_file_relpath)
        base_filename = meta_filename.split('.')[0] + '.'

        # All the table files that correspond to this metadata file
        # (relative to data_dir)

        this_filelist = [
            x for x in table_filelist
            if os.path.dirname(x) == base_relpath and os.path.basename(x).startswith(base_filename)
        ]

        # The 'versions' dict that will go into the metadata
        version_info = {}

        # Make sure function types are the same
        function_types = None

        # For each table basis, compose it
        for table_file in this_filelist:
            # Obtain just the filename of the table basis
            table_filename = os.path.basename(table_file)

            # Obtain the base filename and version from the filename
            # The base filename is the part before the first period
            # (filebase.ver.table.json)
            table_filebase, ver, _, _ = table_filename.split('.')

            # Fully compose the basis set from components
            bs = compose_table_basis(table_file, data_dir)

            # Elements for which this basis is defined
            defined_elements = sorted(list(bs['elements'].keys()), key=lambda x: int(x))

            # Determine the types of functions contained in the basis
            # (gto, ecp, etc)
            if function_types is None:
                function_types = bs['function_types']
            elif function_types != bs['function_types']:
                raise RuntimeError("Differing function types across versions for " + base_filename)

            # Create the metadata for this specific version
            # yapf: disable
            version_info[ver] = { 'file_relpath': table_file,
                                  'revdesc': bs['revision_description'],
                                  'elements': defined_elements
                                }
            # yapf: enable

        # Sort the version dicts
        version_info = dict(sorted(version_info.items()))
        # Find the maximum version for this basis
        latest_ver = max(version_info.keys())

        # Create the common metadata for this basis set
        # display_name and other_names are placeholders to keep order
        # yapf: disable
        common_md = { 'display_name': None,
                      'other_names': None,
                      'description': bs['description'],
                      'latest_version': latest_ver,
                      'basename': base_filename[:-1], # Strip off that trailing period
                      'relpath': base_relpath,
                      'family': bs['family'],
                      'role': bs['role'],
                      'functiontypes': function_types,
                      'auxiliaries': bs['auxiliaries'],
                      'versions': version_info }
        # yapf: enable

        # Loop through all the common names, translate them, and then add the data
        for bs_name in bs_metadata['names']:
            tr_name = transform_basis_name(bs_name)

            if tr_name in metadata:
                raise RuntimeError("Duplicate basis set name: " + tr_name)

            # Create a new entry, with all the common metadata
            # Also, store the other names for this basis
            other_names = bs_metadata['names'].copy()
            other_names.remove(bs_name)
            metadata[tr_name] = common_md.copy()
            metadata[tr_name]['display_name'] = bs_name
            metadata[tr_name]['other_names'] = other_names

    # Write out the metadata
    metadata = dict(sorted(metadata.items()))
    _write_plain_json(output_path, metadata)
873592
def _download_and_clean_file(filename, url):
  """"""
  temp_file, _ = urllib.request.urlretrieve(url)
  with tf.gfile.Open(temp_file, 'r') as temp_eval_file:
    with tf.gfile.Open(filename, 'w') as eval_file:
      for line in temp_eval_file:
        line = line.strip()
        line = line.replace(', ', ',')
        if not line or ',' not in line:
          continue
        if line[-1] == '.':
          line = line[:-1]
        line += '\n'
        eval_file.write(line)
  tf.gfile.Remove(temp_file)
418041
def make_tables(job_dict):
        """"""
        col_dbkey = Column(name='dbkey', dtype=int)
        col_jobname = Column(name='jobname', dtype='S64')
        col_jobkey = Column(name='jobkey', dtype='S64')
        col_appname = Column(name='appname', dtype='S64')
        col_logfile = Column(name='logfile', dtype='S256')
        col_job_config = Column(name='job_config', dtype='S1024')
        col_timestamp = Column(name='timestamp', dtype=int)
        col_infile_refs = Column(name='infile_refs', dtype=int, shape=(2))
        col_outfile_refs = Column(name='outfile_refs', dtype=int, shape=(2))
        col_rmfile_refs = Column(name='rmfile_refs', dtype=int, shape=(2))
        col_intfile_refs = Column(name='intfile_refs', dtype=int, shape=(2))
        col_status = Column(name='status', dtype=int)
        columns = [col_dbkey, col_jobname, col_jobkey, col_appname,
                   col_logfile, col_job_config, col_timestamp,
                   col_infile_refs, col_outfile_refs,
                   col_rmfile_refs, col_intfile_refs,
                   col_status]

        table = Table(data=columns)

        col_file_ids = Column(name='file_id', dtype=int)
        table_ids = Table(data=[col_file_ids])

        for val in job_dict.values():
            val.append_to_tables(table, table_ids)
        return table, table_ids
28109
def format(self, format, dependencies = 0, limit_command_line = None):
    """"""
    command_line = self._cmdline()
    if limit_command_line is not None and len(command_line) > limit_command_line:
      command_line = command_line[:limit_command_line-3] + '...'

    job_id = "%d" % self.id + (" [%d-%d:%d]" % self.get_array() if self.array else "")
    status = "%s" % self.status + (" (%d)" % self.result if self.result is not None else "" )
    queue = self.queue_name if self.machine_name is None else self.machine_name
    if limit_command_line is None:
      grid_opt = self.get_arguments()
      if grid_opt:
        # add additional information about the job at the end
        command_line = "<" + ",".join(["%s=%s" % (key,value) for key,value in grid_opt.items()]) + ">: " + command_line
      if self.exec_dir is not None:
        command_line += "; [Executed in directory: '%s']" % self.exec_dir

    if dependencies:
      deps = str(sorted(list(set([dep.unique for dep in self.get_jobs_we_wait_for()]))))
      if dependencies < len(deps):
        deps = deps[:dependencies-3] + '...'
      return format.format(self.unique, job_id, queue[:12], status, self.name, deps, command_line)
    else:
      return format.format(self.unique, job_id, queue[:12], status, self.name, command_line)
1253540
def _createJobStateFile(self):
        
        jobStateFile = os.path.join(self.localTempDir, '.jobState')
        jobState = {'jobPID': os.getpid(),
                    'jobName': self.jobName,
                    'jobDir': self.localTempDir,
                    'deferredFunctions': []}
        with open(jobStateFile + '.tmp', 'wb') as fH:
            dill.dump(jobState, fH)
        os.rename(jobStateFile + '.tmp', jobStateFile)
        return jobStateFile
262251
def create_on_task(self, task_id, file_content, file_name, file_content_type=None, **options):
        """"""
        path = '/tasks/%d/attachments' % (task_id)
        return self.client.request('post', path, files=[('file', (file_name, file_content, file_content_type))], **options)
636428
def download(csvpath, asset_manager_id, data_id_type, data_id_list):
        """"""
        interface = interface_direct_csvpath(csvpath)
        logging.config.dictConfig(DEFAULT_LOGGING)
        logger = logging.getLogger(__name__)
        objs = []
        for data_id in data_id_list:
            Dict = dict()
            Dict[data_id_type] = data_id
            objs.append(interface.retrieve(asset_manager_id=asset_manager_id, **Dict))
        return objs
1024169
def setup_jobs(outpath, options, input_files):
    ''' 
    '''

    job_inputs = None
    reverse_mapping = None
    fasta_file_contents = {}

    # Generate FASTA files for PDB inputs
    # fasta_file_contents is a mapping from a file path to a pair (FASTA contents, file type). We remember the file type
    # since we offset residue IDs depending on file type i.e. for FASTA files, we treat each sequence separately and do
    # not renumber the fragments in postprocessing. For PDB files, however, we need to respect the order and length of
    # sequences so that we renumber the fragments appropriately in postprocessing - we assume that if a PDB file is passed in
    # then all chains (protein, RNA, or DNA) will be used in a Rosetta run.
    for input_file in input_files:
        assert(not(fasta_file_contents.get(input_file)))
        if any(fnmatch(input_file, x) for x in pdb_file_wildcards):
            pdb = PDB.from_filepath(input_file, strict=True)
            pdb.pdb_id = os.path.basename(input_file).split('.')[0]            
            if pdb.pdb_id.startswith('pdb') and len(pdb.pdb_id) >= 7:
                # Hack to rename FASTA identifiers for pdb*.ent files which are present in mirrors of the PDB
                pdb.pdb_id = pdb.pdb_id.replace('pdb', '')    
            fasta_file_contents[input_file] = (pdb.create_fasta(prefer_seqres_order = False), 'PDB')
        else:
            fasta_file_contents[input_file] = (read_file(input_file), 'FASTA')

    # Extract sequences from the input FASTA files.
    found_sequences, reverse_mapping, errors = get_sequences(options, fasta_file_contents)
    if found_sequences:
        reformat(found_sequences)
    if errors:
        return None, False, errors

    # Discard sequences that are the wrong chain.
    desired_sequences = {}
    for key, sequence in found_sequences.iteritems():
        pdb_id, chain, file_name = key
        if options.chain is None or chain == options.chain:
            desired_sequences[key] = sequence

    # Create the input FASTA and script files.
    job_inputs, errors = create_inputs(options, outpath, desired_sequences)

    # Create the reverse mapping file
    if reverse_mapping:
        segment_mapping_file = os.path.join(outpath, "segment_map.json")
        colorprinter.message("Creating a reverse mapping file %s." % segment_mapping_file)
        write_file(segment_mapping_file, json.dumps(reverse_mapping))

    # Create the post-processing script file
    post_processing_script = read_file(os.path.join(os.path.split(os.path.realpath(__file__))[0], 'post_processing.py'))
    write_file(os.path.join(outpath, 'post_processing.py'), post_processing_script, 'w')

    # Create the secondary structure filter file
    if options.secondary_structure_file:
        write_file(os.path.join(outpath, 'ss_filter.json'), json.dumps({'secondary_structure_filter' : SecondaryStructureDefinition.from_filepath(options.secondary_structure_file).data}), 'w')

    return job_inputs, reverse_mapping != None, errors