Hook to build job configurations
496163
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])
        NAME_FACTORY.update_base_dict(args['data'])

        ret_dict = make_diffuse_comp_info_dict(components=components,
                                               library=args['library'],
                                               basedir=NAME_FACTORY.base_dict['basedir'])
        diffuse_comp_info_dict = ret_dict['comp_info_dict']

        for diffuse_comp_info_key in sorted(diffuse_comp_info_dict.keys()):
            diffuse_comp_info_value = diffuse_comp_info_dict[diffuse_comp_info_key]

            for comp in components:
                zcut = "zmax%i" % comp.zmax
                key = comp.make_key('{ebin_name}_{evtype_name}')
                if diffuse_comp_info_value.components is None:
                    sub_comp_info = diffuse_comp_info_value
                else:
                    sub_comp_info = diffuse_comp_info_value.get_component_info(comp)
                name_keys = dict(zcut=zcut,
                                 sourcekey=sub_comp_info.sourcekey,
                                 ebin=comp.ebin_name,
                                 psftype=comp.evtype_name,
                                 mktime='none',
                                 coordsys=comp.coordsys,
                                 irf_ver=NAME_FACTORY.irf_ver(),
                                 fullpath=True)

                outfile = NAME_FACTORY.srcmaps(**name_keys)
                outfile_tokens = os.path.splitext(outfile)
                infile_regexp = "%s_*.fits*" % outfile_tokens[0]
                full_key = "%s_%s" % (sub_comp_info.sourcekey, key)
                logfile = make_nfs_path(outfile.replace('.fits', '.log'))
                job_configs[full_key] = dict(output=outfile,
                                             args=infile_regexp,
                                             hdu=sub_comp_info.source_name,
                                             logfile=logfile)

        return job_configs
764423
public void configure(JobConf job) {
    this.longCounters = new TreeMap<Object, Long>();
    this.doubleCounters = new TreeMap<Object, Double>();
  }
709207
def present(name,
            config=None,
            **kwargs):
    '''
    
    '''

    ret = {'name': name,
           'result': True,
           'changes': {},
           'comment': ['Job {0} is up to date.'.format(name)]}

    if __salt__['jenkins.job_exists'](name):
        _current_job_config = __salt__['jenkins.get_job_config'](name)
        buf = six.moves.StringIO(_current_job_config)
        oldXML = ET.fromstring(buf.read())

        cached_source_path = __salt__['cp.cache_file'](config, __env__)
        with salt.utils.files.fopen(cached_source_path) as _fp:
            newXML = ET.fromstring(salt.utils.stringutils.to_unicode(_fp.read()))
        if not _elements_equal(oldXML, newXML):
            diff = difflib.unified_diff(
                ET.tostringlist(oldXML, encoding='utf8', method='xml'),
                ET.tostringlist(newXML, encoding='utf8', method='xml'), lineterm='')
            try:
                __salt__['jenkins.update_job'](name, config, __env__)
            except CommandExecutionError as exc:
                return _fail(ret, exc.strerror)
            else:
                ret['changes'] = ''.join(diff)
                ret['comment'].append('Job \'{0}\' updated.'.format(name))

    else:
        cached_source_path = __salt__['cp.cache_file'](config, __env__)
        with salt.utils.files.fopen(cached_source_path) as _fp:
            new_config_xml = salt.utils.stringutils.to_unicode(_fp.read())

        try:
            __salt__['jenkins.create_job'](name, config, __env__)
        except CommandExecutionError as exc:
            return _fail(ret, exc.strerror)

        buf = six.moves.StringIO(new_config_xml)
        diff = difflib.unified_diff('', buf.readlines(), lineterm='')
        ret['changes'][name] = ''.join(diff)
        ret['comment'].append('Job \'{0}\' added.'.format(name))

    ret['comment'] = '\n'.join(ret['comment'])
    return ret
1272791
public static void configureDB(JobConf job, String driverClass, String dbUrl) {
    configureDB(job, driverClass, dbUrl, null, null);
  }
1310930
def _hook(self, hook_name, doc_uri=None, **kwargs):
        """"""
        doc = self.workspace.get_document(doc_uri) if doc_uri else None
        hook_handlers = self.config.plugin_manager.subset_hook_caller(hook_name, self.config.disabled_plugins)
        return hook_handlers(config=self.config, workspace=self.workspace, document=doc, **kwargs)
419007
function (jobName, param, customParams, callback) {
      [jobName, param, customParams, callback] = doArgs(arguments, ['string', ['string', 'id,timestamp,result,duration'], ['object', {}], 'function']);

      // TODO better name and handle the "param" ???
      doRequest({
        urlPattern: [ALL_BUILDS, jobName, param],
        bodyProp: 'allBuilds'
      }, customParams, callback);
    }
415610
def mk_travis_config():
    """"""
    t = dedent("""\
        language: python
        python: 3.4
        env:
        {jobs}
        install:
            - pip install -r requirements/ci.txt
        script:
            - invoke ci_run_job $TOX_JOB
        after_success:
            coveralls
    """)
    jobs = [env for env in parseconfig(None, 'tox').envlist
            if not env.startswith('cov-')]
    jobs += 'coverage',
    print(t.format(jobs=('\n'.join(('    - TOX_JOB=' + job)
                                   for job in jobs))))
338700
def build_job_configs(self, args):
        
        job_configs = {}

        components = Component.build_from_yamlfile(args['comp'])

        datafile = args['data']
        if datafile is None or datafile == 'None':
            return job_configs
        NAME_FACTORY.update_base_dict(args['data'])
        outdir_base = os.path.join(NAME_FACTORY.base_dict['basedir'], 'counts_cubes')

        inputfiles = create_inputlist(args['ft1file'])
        num_files = len(inputfiles)

        for comp in components:
            zcut = "zmax%i" % comp.zmax

            mktimelist = copy.copy(comp.mktimefilters)
            if not mktimelist:
                mktimelist.append('none')
            evtclasslist_keys = copy.copy(comp.evtclasses)
            if not evtclasslist_keys:
                evtclasslist_vals = [NAME_FACTORY.base_dict['evclass']]
            else:
                evtclasslist_vals = copy.copy(evtclasslist_keys)

            for mktimekey in mktimelist:
                for evtclassval in evtclasslist_vals:
                    fullkey = comp.make_key(
                        '%s_%s_{ebin_name}_%s_{evtype_name}' %
                        (evtclassval, zcut, mktimekey))

                    name_keys = dict(zcut=zcut,
                                     ebin=comp.ebin_name,
                                     psftype=comp.evtype_name,
                                     coordsys=comp.coordsys,
                                     irf_ver=NAME_FACTORY.irf_ver(),
                                     mktime=mktimekey,
                                     evclass=evtclassval,
                                     fullpath=True)

                    ccube_name = os.path.basename(NAME_FACTORY.ccube(**name_keys))
                    outfile = os.path.join(outdir_base, ccube_name)
                    infiles = _make_input_file_list(outfile, num_files)
                    logfile = make_nfs_path(outfile.replace('.fits', '.log'))
                    job_configs[fullkey] = dict(args=infiles,
                                                output=outfile,
                                                logfile=logfile)

        return job_configs
966789
def _run_hooks(config, hooks, args, environ):
    """"""
    skips = _get_skips(environ)
    cols = _compute_cols(hooks, args.verbose)
    filenames = _all_filenames(args)
    filenames = filter_by_include_exclude(filenames, '', config['exclude'])
    classifier = Classifier(filenames)
    retval = 0
    for hook in hooks:
        retval |= _run_single_hook(classifier, hook, args, skips, cols)
        if retval and config['fail_fast']:
            break
    if retval and args.show_diff_on_failure and git.has_diff():
        if args.all_files:
            output.write_line(
                'pre-commit hook(s) made changes.\n'
                'If you are seeing this message in CI, '
                'reproduce locally with: `pre-commit run --all-files`.\n'
                'To run `pre-commit` as part of git workflow, use '
                '`pre-commit install`.',
            )
        output.write_line('All changes made by hooks:')
        subprocess.call(('git', '--no-pager', 'diff', '--no-ext-diff'))
    return retval
1244897
function (jobName, buildNumber, customParams, callback) {
      [jobName, buildNumber, customParams, callback] = doArgs(arguments, ['string', 'string|number', ['object', {}], 'function']);

      doRequest({
        urlPattern: [JOB_OUTPUT, jobName, buildNumber],
        noparse: true
      }, customParams, callback);
    }
118871
def _setup_fun_config(self, fun_conf):
        '''
        
        '''
        conf = copy.deepcopy(self.config)
        conf['file_client'] = 'local'
        conf['fun'] = ''
        conf['arg'] = []
        conf['kwarg'] = {}
        conf['cache_jobs'] = False
        conf['print_metadata'] = False
        conf.update(fun_conf)
        conf['fun'] = conf['fun'].split(':')[-1]  # Discard typing prefix

        return conf
832532
def build_job_configs(self, args):
        
        job_configs = {}

        ttype = args['ttype']
        (sim_targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
        targets = load_yaml(sim_targets_yaml)

        base_config = dict(ttype=ttype,
                           roi_baseline=args['roi_baseline'],
                           extracopy = args['extracopy'],
                           sim=sim)

        for target_name in targets.keys():
            targetdir = NAME_FACTORY.sim_targetdir(target_type=ttype,
                                                   target_name=target_name,
                                                   sim_name=sim)
            logfile = os.path.join(targetdir, 'copy_base_dir.log')
            job_config = base_config.copy()
            job_config.update(dict(target=target_name,
                                   logfile=logfile))
            job_configs[target_name] = job_config

        return job_configs
56273
def _register_jobs(self):
        

        # job_name is hard-corded
        job_name = 'ConcreteJob'
        modules = self._get_modules()

        for section, options in self.config.items():

            if section == 'global':
                continue

            try:
                name = options['module']
            except KeyError:
                raise ConfigMissingValue(section, 'module')

            try:
                job = getattr(modules[name], job_name)
                self.notify(name, job)
            except KeyError:
                raise NotSupportedError(name)
607836
def setup_job(manager, job_id, tool_id, tool_version, use_metadata=False):
    
    job_id = manager.setup_job(job_id, tool_id, tool_version)
    if use_metadata:
        manager.enable_metadata_directory(job_id)
    return build_job_config(
        job_id=job_id,
        job_directory=manager.job_directory(job_id),
        system_properties=manager.system_properties(),
        tool_id=tool_id,
        tool_version=tool_version
    )
220272
def build_job_configs(self, args):
        
        job_configs = {}

        ttype = args['ttype']
        (targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
        if targets_yaml is None:
            return job_configs

        config_yaml = 'config.yaml'
        config_override = args.get('config')
        if is_not_null(config_override):
            config_yaml = config_override

        targets = load_yaml(targets_yaml)
        nsims_job = args['nsims_job']
        first_seed = args['seed']
        nsims = args['nsims']
        last_seed = first_seed + nsims

        base_config = dict(sim_profile=args['sim_profile'],
                           roi_baseline=args['roi_baseline'],
                           non_null_src=args['non_null_src'],
                           sim=sim)

        for target_name, target_list in targets.items():
            name_keys = dict(target_type=ttype,
                             target_name=target_name,
                             sim_name=sim,
                             fullpath=True)
            simdir = NAME_FACTORY.sim_targetdir(**name_keys)
            config_path = os.path.join(simdir, config_yaml)

            job_config = base_config.copy()
            job_config.update(dict(config=config_path,
                                   profiles=target_list))

            current_seed = first_seed
            while current_seed < last_seed:
                fullkey = "%s_%06i" % (target_name, current_seed)
                logfile = make_nfs_path(os.path.join(simdir, "%s_%s_%06i.log" % (self.linkname, 
                                                                                 target_name, current_seed)))
                if nsims_job <= 0 or current_seed + nsims_job >= last_seed:
                    nsims_current = last_seed - current_seed
                else:
                    nsims_current = nsims_job
                job_config.update(dict(seed=current_seed,
                                       nsims=nsims_current,
                                       logfile=logfile))
                job_configs[fullkey] = job_config.copy()
                current_seed += nsims_current

        return job_configs
1076595
function push(jobProfileString, jobOpt /*jobOptInterface*/, namespace) {
    logger.debug(`Following litteral was pushed \n ${util.format(jobOpt)}`);
    let jobID = uuidv4();
    if (jobOpt.hasOwnProperty('id'))
        if (jobOpt.id)
            jobID = jobOpt.id;
    let workDir;
    if (namespace) {
        try {
            fs.mkdirSync(cacheDir + '/' + namespace);
        }
        catch (err) {
            if (err.code != 'EEXIST') {
                logger.error("Namespace " + cacheDir + '/' + namespace + ' already exists.');
                throw err;
            }
        }
        workDir = cacheDir + '/' + namespace + '/' + jobID;
    }
    else {
        workDir = cacheDir + '/' + jobID;
    }
    /* Building a jobOptInterface litteral out of the jobOpt function parameter */
    let jobTemplate = {
        // "engineHeader": engine.generateHeader(jobID, jobProfileString, workDir),
        "engine": engine,
        "workDir": workDir,
        "emulated": emulator ? true : false,
        "adress": TCPip,
        "port": TCPport,
        "jobProfile": jobProfileString ? jobProfileString : "default"
        // "submitBin": engine.submitBin(),
    };
    if ('exportVar' in jobOpt)
        jobTemplate.exportVar = jobOpt.exportVar;
    if ('modules' in jobOpt)
        jobTemplate.modules = jobOpt.modules;
    if ('script' in jobOpt)
        jobTemplate.script = jobOpt.script;
    if ('cmd' in jobOpt)
        jobTemplate.cmd = jobOpt.cmd;
    if ('inputs' in jobOpt)
        jobTemplate.inputs = jobOpt.inputs;
    if ('tagTask' in jobOpt)
        jobTemplate.tagTask = jobOpt.tagTask;
    if ('ttl' in jobOpt)
        jobTemplate.ttl = jobOpt.ttl;
    if ('socket' in jobOpt)
        jobTemplate.socket = jobOpt.socket;
    logger.debug(`Following jobTemplate was successfully buildt \n ${util.format(jobTemplate)}`);
    let newJob = new jobLib.jobObject(jobTemplate, jobID);
    if ('fromConsumerMS' in jobOpt)
        newJob.fromConsumerMS = jobOpt.fromConsumerMS;
    // 3 outcomes
    // newJob.launch // genuine start
    // newJob.resurrect // load from wareHouse a complete job
    // newJob.melt // replace newJob by an already running job
    //                just copying client socket if fromConsumerMS 
    //
    logger.debug(`Following jobObject was successfully buildt \n ${util.format(newJob)}`);
    newJob.start();
    liveMemory.addJob(newJob);
    newJob.on('inputSet', function () {
        // All input streams were dumped to file(s), we can safely serialize
        let jobSerial = newJob.getSerialIdentity();
        MS_lookup(jobSerial)
            .on('known', function (fStdoutName, fStderrName, workPath) {
            newJob.respawn(fStdoutName, fStderrName, workPath);
            newJob.jEmit("completed", newJob);
            // Repertoir valide
            // Extraire la sortie d'erreur / standard du job et faire lever levent completed
            //logger.info("I CAN RESURRECT YOU : " + validWorkFolder + ' -> ' + jobTemplate.tagTask);
            //_resurrect(newJob, validWorkFolder);
        })
            .on('unknown', function () {
            logger.debug("####No suitable job found in warehouse");
            let previousJobs;
            previousJobs = liveMemory.lookup(newJob);
            if (previousJobs) {
                // let refererJob:jobLib.jobObject = getJob(previousJobs[0].id];
                logger.debug(`${previousJobs.length} suitable living job(s) found, shimmering`);
                melting(previousJobs[0], newJob);
                return;
            }
            logger.debug('No Suitable living jobs found, launching');
            //liveStore(newJob.getSerialIdentity());  
            //jobRegister(newJob);
            liveMemory.jobSet('source', { jobObject: newJob });
            newJob.launch();
            newJob.on('submitted', function (j) {
                liveMemory.jobSet('SUBMITTED', { jobObject: newJob });
                //jobsArray[j.id].status = 'SUBMITTED';
            }).on('jobStart', function (job) {
                engine.list();
                // shall we call dropJob function here ?
            }).on('scriptReadError', function (err, job) {
                logger.error(`ERROR while reading the script : \n ${err}`);
            }).on('scriptWriteError', function (err, job) {
                logger.error(`ERROR while writing the coreScript : \n ${err}`);
            }).on('scriptSetPermissionError', function (err, job) {
                logger.error(`ERROR while trying to set permissions of the coreScript : \n ${err}`);
            });
        });
    });
    exhaustBool = true;
    //console.log(jobsArray);
    return newJob;
}
991543
def _build_config_tree(name, configuration):
    '''
    
    '''
    type_, id_, options = _get_type_id_options(name, configuration)
    global _INDENT, _current_statement
    _INDENT = ''
    if type_ == 'config':
        _current_statement = GivenStatement(options)
    elif type_ == 'log':
        _current_statement = UnnamedStatement(type='log')
        _parse_log_statement(options)
    else:
        if _is_statement_unnamed(type_):
            _current_statement = UnnamedStatement(type=type_)
        else:
            _current_statement = NamedStatement(type=type_, id=id_)
        _parse_statement(options)