Hook to build job configurations
338700
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])

        datafile = args['data']
        if datafile is None or datafile == 'None':
            return job_configs
        NAME_FACTORY.update_base_dict(args['data'])
        outdir_base = os.path.join(NAME_FACTORY.base_dict['basedir'], 'counts_cubes')

        inputfiles = create_inputlist(args['ft1file'])
        num_files = len(inputfiles)

        for comp in components:
            zcut = "zmax%i" % comp.zmax

            mktimelist = copy.copy(comp.mktimefilters)
            if not mktimelist:
                mktimelist.append('none')
            evtclasslist_keys = copy.copy(comp.evtclasses)
            if not evtclasslist_keys:
                evtclasslist_vals = [NAME_FACTORY.base_dict['evclass']]
            else:
                evtclasslist_vals = copy.copy(evtclasslist_keys)

            for mktimekey in mktimelist:
                for evtclassval in evtclasslist_vals:
                    fullkey = comp.make_key(
                        '%s_%s_{ebin_name}_%s_{evtype_name}' %
                        (evtclassval, zcut, mktimekey))

                    name_keys = dict(zcut=zcut,
                                     ebin=comp.ebin_name,
                                     psftype=comp.evtype_name,
                                     coordsys=comp.coordsys,
                                     irf_ver=NAME_FACTORY.irf_ver(),
                                     mktime=mktimekey,
                                     evclass=evtclassval,
                                     fullpath=True)

                    ccube_name = os.path.basename(NAME_FACTORY.ccube(**name_keys))
                    outfile = os.path.join(outdir_base, ccube_name)
                    infiles = _make_input_file_list(outfile, num_files)
                    logfile = make_nfs_path(outfile.replace('.fits', '.log'))
                    job_configs[fullkey] = dict(args=infiles,
                                                output=outfile,
                                                logfile=logfile)

        return job_configs
881589
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])
        NAME_FACTORY.update_base_dict(args['data'])

        ret_dict = make_diffuse_comp_info_dict(components=components,
                                               library=args['library'],
                                               basedir='.')
        diffuse_comp_info_dict = ret_dict['comp_info_dict']
        if args['make_xml']:
            SrcmapsDiffuse_SG._make_xml_files(diffuse_comp_info_dict)

        for diffuse_comp_info_key in sorted(diffuse_comp_info_dict.keys()):
            diffuse_comp_info_value = diffuse_comp_info_dict[diffuse_comp_info_key]
            no_psf = diffuse_comp_info_value.no_psf
            for comp in components:
                zcut = "zmax%i" % comp.zmax
                key = comp.make_key('{ebin_name}_{evtype_name}')
                if diffuse_comp_info_value.components is None:
                    sub_comp_info = diffuse_comp_info_value
                else:
                    sub_comp_info = diffuse_comp_info_value.get_component_info(comp)
                name_keys = dict(zcut=zcut,
                                 sourcekey=sub_comp_info.sourcekey,
                                 ebin=comp.ebin_name,
                                 psftype=comp.evtype_name,
                                 mktime='none',
                                 coordsys=comp.coordsys,
                                 irf_ver=NAME_FACTORY.irf_ver(),
                                 fullpath=True)

                kmin = 0
                kmax = comp.enumbins + 1
                outfile_base = NAME_FACTORY.srcmaps(**name_keys)
                kstep = HPX_ORDER_TO_KSTEP[comp.hpx_order]
                base_dict = dict(cmap=NAME_FACTORY.ccube(**name_keys),
                                 expcube=NAME_FACTORY.ltcube(**name_keys),
                                 irfs=NAME_FACTORY.irfs(**name_keys),
                                 bexpmap=NAME_FACTORY.bexpcube(**name_keys),
                                 srcmdl=sub_comp_info.srcmdl_name,
                                 source=sub_comp_info.source_name,
                                 no_psf=no_psf,
                                 evtype=comp.evtype)

                if kstep < 0:
                    kstep = kmax
                else:
                    pass

                for k in range(kmin, kmax, kstep):
                    full_key = "%s_%s_%02i" % (diffuse_comp_info_key, key, k)
                    khi = min(kmax, k + kstep)

                    full_dict = base_dict.copy()
                    outfile = outfile_base.replace('.fits', '_%02i.fits' % k)
                    logfile = make_nfs_path(outfile_base.replace('.fits', '_%02i.log' % k))
                    full_dict.update(dict(outfile=outfile,
                                          kmin=k, kmax=khi,
                                          logfile=logfile))
                    job_configs[full_key] = full_dict

        return job_configs
134411
private static JobConf createJobConf(Configuration conf) {
    JobConf jobconf = new JobConf(conf, DistRaid.class);
    jobName = NAME + " " + dateForm.format(new Date(RaidNode.now()));
    jobconf.setUser(RaidNode.JOBUSER);
    jobconf.setJobName(jobName);
    jobconf.setMapSpeculativeExecution(false);
    RaidUtils.parseAndSetOptions(jobconf, SCHEDULER_OPTION_LABEL);

    jobconf.setJarByClass(DistRaid.class);
    jobconf.setInputFormat(DistRaidInputFormat.class);
    jobconf.setOutputKeyClass(Text.class);
    jobconf.setOutputValueClass(Text.class);

    jobconf.setMapperClass(DistRaidMapper.class);
    jobconf.setNumReduceTasks(0);
    return jobconf;
  }
137675
def generate(self, job):
        

        options = job.get_options().copy()
        job_name = options.pop('name', None)
        job_account = options.pop('account', None)
        job_walltime = options.pop('walltime', None)
        job_mem_per_cpu = options.pop('mem_per_cpu', None)
        job_memory = options.pop('memory', None)
        job_working_directory = options.pop('working_directory', None)
        job_error_path = options.pop('error_path', None)
        job_output_path = options.pop('output_path', None)
        job_dependency = options.pop('depending', None)
        job_shell = options.pop('shell', '/bin/bash')
        job_custom_options = options.pop('__custom__', [])

        directives_lines = []

        if job_name is not None:
            directives_lines.append('--job-name=%s' % job_name)

        if job_account is not None:
            directives_lines.append('--account=%s' % job_account)

        if job_working_directory is not None:
            directives_lines.append('--workdir=%s' % job_working_directory)

        if job_error_path is not None:
            directives_lines.append('--error=%s' % job_error_path)

        if job_output_path is not None:
            directives_lines.append('--output=%s' % job_output_path)

        if job_walltime is not None:
            directives_lines.append('--time=%s' %
                                    strfdelta(job_walltime, '%H:%M:%S'))

        if job_mem_per_cpu is not None:
            directives_lines.append('--mem-per-cpu=%d' % job_mem_per_cpu)

        if job_memory is not None:
            directives_lines.append('--mem=%d' % job_memory)

        if job_dependency is not None:
            master = job_dependency['job']
            dependency_type = SlurmPrinter.get_dependency_type(
                job_dependency['dependency_type']
            )
            job_id = master.get_id() if isinstance(master, JobInterface) else master

            directives_lines.append(
                '--dependency=%s:%s' %
                (dependency_type, job_id)
            )

        for custom_option in job_custom_options:
            directives_lines.append(custom_option)

        directives = '\n'.join([
            '#SBATCH %s' % directive for directive in directives_lines
        ])

        commands = '\n'.join([
            '\n'.join(command_container.get_commands()) for command_container in job.get_commands()
        ])

        script = '#!%s\n' % job_shell
        script += SlurmPrinter.get_header()
        script += directives
        script += '\n\n'
        script += commands

        return script
751177
def build_configuration_parameters(app):
    """"""
    env = Environment(loader=FileSystemLoader("{0}/_data_templates".format(BASEPATH)))
    template_file = env.get_template("configuration-parameters.j2")
    data = {}
    data["schema"] = Config.schema()
    rendered_template = template_file.render(**data)
    output_dir = "{0}/configuration/generated".format(BASEPATH)
    with open("{}/parameters.rst".format(output_dir), "w") as f:
        f.write(rendered_template)
806368
def build(client, destination_args):
    
    # Have defined a remote job directory, lets do the setup locally.
    if client.job_directory:
        handler = LocalSetupHandler(client, destination_args)
    else:
        handler = RemoteSetupHandler(client)
    return handler
1229548
function (jobName, modifyFunction, customParams, callback) {
      [jobName, modifyFunction, customParams, callback] = doArgs(arguments, ['string', 'function', ['object', {}], 'function']);

      const self = this;

      self.get_config_xml(jobName, customParams, function (error, data) {
        if (error) {
          callback(error, data);
          return;
        }

        // Modify the config data
        data = modifyFunction(data);

        self.update_job(jobName, data, customParams, callback);
      });
    }
453167
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])
        datafile = args['data']
        if datafile is None or datafile == 'None':
            return job_configs
        NAME_FACTORY.update_base_dict(args['data'])

        for comp in components:
            zcut = "zmax%i" % comp.zmax

            mktimelist = copy.copy(comp.mktimefilters)
            if not mktimelist:
                mktimelist.append('none')
            evtclasslist_keys = copy.copy(comp.evtclasses)
            if not evtclasslist_keys:
                evtclasslist_vals = [NAME_FACTORY.base_dict['evclass']]
            else:
                evtclasslist_vals = copy.copy(evtclasslist_keys)

            for mktimekey in mktimelist:
                for evtclassval in evtclasslist_vals:
                    fullkey = comp.make_key(
                        '%s_%s_{ebin_name}_%s_{evtype_name}' %
                        (evtclassval, zcut, mktimekey))
                    name_keys = dict(zcut=zcut,
                                     ebin=comp.ebin_name,
                                     psftype=comp.evtype_name,
                                     coordsys=comp.coordsys,
                                     irf_ver=NAME_FACTORY.irf_ver(),
                                     mktime=mktimekey,
                                     evclass=evtclassval,
                                     fullpath=True)

                    outfile = NAME_FACTORY.bexpcube(**name_keys)
                    cmap = NAME_FACTORY.ccube(**name_keys)
                    infile = NAME_FACTORY.ltcube(**name_keys)
                    logfile = make_nfs_path(outfile.replace('.fits', '.log'))
                    job_configs[fullkey] = dict(cmap=cmap,
                                                infile=infile,
                                                outfile=outfile,
                                                irfs=NAME_FACTORY.irfs(**name_keys),
                                                hpx_order=min(
                                                    comp.hpx_order, args['hpx_order_max']),
                                                evtype=comp.evtype,
                                                logfile=logfile)

        return job_configs
1005571
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])
        NAME_FACTORY.update_base_dict(args['data'])

        # FIXME
        mktime = args['mktimefilter']

        for comp in components:
            for sourcekey in args['sourcekeys']:
                zcut = "zmax%i" % comp.zmax
                key = comp.make_key('{ebin_name}_{evtype_name}') + "_%s" % sourcekey
                name_keys = dict(zcut=zcut,
                                 ebin=comp.ebin_name,
                                 psftype=comp.evtype_name,
                                 irf_ver=NAME_FACTORY.irf_ver(),
                                 sourcekey=sourcekey,
                                 mktime=mktime,
                                 coordsys=comp.coordsys,
                                 fullpath=True)
                outfile = NAME_FACTORY.template_sunmoon(**name_keys)
                job_configs[key] = dict(expsun=NAME_FACTORY.bexpcube_sun(**name_keys),
                                        avgexp=NAME_FACTORY.bexpcube(**name_keys),
                                        sunprof=NAME_FACTORY.angprofile(**name_keys),
                                        cmap='none',
                                        outfile=outfile,
                                        irfs=NAME_FACTORY.irfs(**name_keys),
                                        evtype=comp.evtype,
                                        emin=comp.emin,
                                        emax=comp.emax,
                                        enumbins=comp.enumbins,
                                        logfile=outfile.replace('.fits', '.log'))

        return job_configs
25628
def default_hook(config):
    """"""
    if (any(arg.startswith('bdist') for arg in sys.argv) and
            os.path.isdir(PY2K_DIR) != IS_PY2K and os.path.isdir(LIB_DIR)):
        shutil.rmtree(LIB_DIR)

    if IS_PY2K and any(arg.startswith('install') or
                       arg.startswith('build') or
                       arg.startswith('bdist') for arg in sys.argv):
        generate_py2k(config)
        packages_root = get_cfg_value(config, 'files', 'packages_root')
        packages_root = os.path.join(PY2K_DIR, packages_root)
        set_cfg_value(config, 'files', 'packages_root', packages_root)
1104612
def hooks_factory(identifier, configuration, context):
    
    manager = HookManager(identifier, configuration)
    manager.load_hooks(context)

    return manager
405070
def _find_travis_job(self, build, toxenv):
        """"""
        for jobid in build.job_ids:
            j = self._travis.job(jobid)
            if 'TOXENV=%s' % toxenv in j.config['env']:
                logger.debug('Found %s job: %s', toxenv, j.number)
                return j
        raise SystemExit(
            'ERROR: could not find %s job for build %s (%s)' % (
                toxenv, build.number, build.id
            )
        )
733852
protected function gatherHookDocumentationFiles($job_list) {
    $directory = \DrupalCodeBuilder\Factory::getEnvironment()->getHooksDirectory();

    // Fetch data about the files we need to download.
    $hook_files = $this->getHookFileUrls($directory);
    //print_r($hook_files);

    // Retrieve each file and store it in the hooks directory, overwriting what's currently there
    foreach ($hook_files as $file_name => $data) {
      $file_contents = drupal_http_request($data['url']);

      // TODO: replace with call to environment output.
      file_put_contents("$directory/$file_name", $destination . $file_contents->data);
    }

    return $hook_files;
  }
1272791
public static void configureDB(JobConf job, String driverClass, String dbUrl) {
    configureDB(job, driverClass, dbUrl, null, null);
  }
1248799
def spawn_batch_jobs(job, shared_ids, input_args):
    
    samples = []
    config = input_args['config']
    with open(config, 'r') as f_in:
        for line in f_in:
            line = line.strip().split(',')
            uuid = line[0]
            urls = line[1:]
            samples.append((uuid, urls))
    for sample in samples:
        job.addChildJobFn(alignment, shared_ids, input_args, sample, cores=32, memory='20 G', disk='100 G')
580701
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])
        NAME_FACTORY.update_base_dict(args['data'])

        mktime = args['mktimefilter']

        for comp in components:
            zcut = "zmax%i" % comp.zmax
            key = comp.make_key('{ebin_name}_{evtype_name}')
            name_keys = dict(zcut=zcut,
                             ebin=comp.ebin_name,
                             psftype=comp.evtype_name,
                             irf_ver=NAME_FACTORY.irf_ver(),
                             mktime=mktime,
                             fullpath=True)
            outfile = NAME_FACTORY.bexpcube_sun(**name_keys)
            ltcube_sun = NAME_FACTORY.ltcube_sun(**name_keys)
            job_configs[key] = dict(infile=NAME_FACTORY.ltcube_sun(**name_keys),
                                    outfile=outfile,
                                    irfs=NAME_FACTORY.irfs(**name_keys),
                                    evtype=comp.evtype,
                                    emin=comp.emin,
                                    emax=comp.emax,
                                    enumbins=comp.enumbins,
                                    logfile=make_nfs_path(outfile.replace('.fits', '.log')))

        return job_configs
832532
def build_job_configs(self, args):
        
        job_configs = {}

        ttype = args['ttype']
        (sim_targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
        targets = load_yaml(sim_targets_yaml)

        base_config = dict(ttype=ttype,
                           roi_baseline=args['roi_baseline'],
                           extracopy = args['extracopy'],
                           sim=sim)

        for target_name in targets.keys():
            targetdir = NAME_FACTORY.sim_targetdir(target_type=ttype,
                                                   target_name=target_name,
                                                   sim_name=sim)
            logfile = os.path.join(targetdir, 'copy_base_dir.log')
            job_config = base_config.copy()
            job_config.update(dict(target=target_name,
                                   logfile=logfile))
            job_configs[target_name] = job_config

        return job_configs