creates a list of Job objects,
    which carry all information needed
    for a function to be executed on SGE:
    - function object
    - arguments
    - settings
970038
def make_jobs():
    

    # set up list of arguments
    inputvec = [[3], [5], [10], [20]]

    # create empty job vector
    jobs = []

    # create job objects
    for arg in inputvec:
        # The default queue used by the Job class is all.q. You must specify
        # the `queue` keyword argument if that is not the name of your queue.
        job = Job(compute_factorial, arg, queue='all.q')
        jobs.append(job)

    return jobs
158785
def queue_raw_jobs(queue, params_list, **kwargs):
    

    from .queue import Queue
    queue_obj = Queue(queue)
    queue_obj.enqueue_raw_jobs(params_list, **kwargs)
1028225
def build_job_configs(self, args):
        
        job_configs = {}

        ttype = args['ttype']
        (targets_yaml, sim) = NAME_FACTORY.resolve_targetfile(args)
        if targets_yaml is None:
            return job_configs

        targets = load_yaml(targets_yaml)

        for target_name, target_list in targets.items():
            for targ_prof in target_list:
                name_keys = dict(target_type=ttype,
                                 target_name=target_name,
                                 profile=targ_prof,
                                 fullpath=True)
                targ_key = "%s_%s" % (target_name, targ_prof)
                input_path = NAME_FACTORY.sedfile(**name_keys)
                output_path = input_path.replace('.fits', '.png')
                logfile = make_nfs_path(input_path.replace('.fits', '.log'))
                job_config = dict(infile=input_path,
                                  outfile=output_path,
                                  logfile=logfile)
                job_configs[targ_key] = job_config

        return job_configs
675642
def build_from_job_list(scheme_files, templates, base_output_dir):
    """"""
    queue = Queue()
    for scheme in scheme_files:
        queue.put(scheme)

    if len(scheme_files) < 40:
        thread_num = len(scheme_files)
    else:
        thread_num = 40

    threads = []
    for _ in range(thread_num):
        thread = Thread(target=build_single_worker,
                        args=(queue, templates, base_output_dir))
        thread.start()
        threads.append(thread)

    queue.join()

    for _ in range(thread_num):
        queue.put(None)

    for thread in threads:
        thread.join()
119837
def apply_args(job, inputs, optional_inputs=None):
    
    _apply_args_loop(job, inputs, INPUT_FIELD)
    _apply_args_loop(job, optional_inputs, OPTIONAL_FIELD)
    return job
705820
def job(func_or_queue=None, connection=None, *args, **kwargs):
    
    if callable(func_or_queue):
        func = func_or_queue
        queue = 'default'
    else:
        func = None
        queue = func_or_queue or 'default'

    if not isinstance(queue, basestring):
        queue = unicode(queue)

    try:
        queue = get_queue(queue)
        if connection is None:
            connection = queue.connection
    except KeyError:
        pass

    decorator = _job(queue, connection=connection, *args, **kwargs)
    if func:
        return decorator(func)
    return decorator
568569
def build_job_configs(self, args):
        
        job_configs = {}

        comp_file = args.get('comp', None)
        if comp_file is not None:
            comp_dict = yaml.safe_load(open(comp_file))
            coordsys = comp_dict.pop('coordsys')
            for v in comp_dict.values():
                v['coordsys'] = coordsys
        else:
            return job_configs

        datafile = args['data']
        if datafile is None or datafile == 'None':
            return job_configs
        NAME_FACTORY.update_base_dict(args['data'])

        inputfiles = create_inputlist(args['ft1file'])
        outdir_base = os.path.join(NAME_FACTORY.base_dict['basedir'], 'counts_cubes')
        data_ver = NAME_FACTORY.base_dict['data_ver']

        for idx, infile in enumerate(inputfiles):
            key = "%06i" % idx
            key_scfile = "%03i" % (idx + 1)
            output_dir = os.path.join(outdir_base, key)
            try:
                os.mkdir(output_dir)
            except OSError:
                pass
            scfile = args['ft2file'].replace('.lst', '_%s.fits' % key_scfile)
            logfile = make_nfs_path(os.path.join(output_dir,
                                                 'scatter_mk_%s_%s.log' % (data_ver, key)))

            job_configs[key] = comp_dict.copy()
            job_configs[key].update(dict(ft1file=infile,
                                         scfile=scfile,
                                         comp=args['comp'],
                                         hpx_order_max=args['hpx_order_max'],
                                         outdir=outdir_base,
                                         outkey=key,
                                         logfile=logfile,
                                         pfiles=output_dir))

        return job_configs
1125480
def get_plugins(self, shell_params, job_params):
        
        shell = self.get_shell_plugin(shell_params)
        job_interface = self.get_job_interface(job_params)
        return shell, job_interface
417441
function build_info(jobName, buildNumber, customParams, callback) {
      var _doArgs15 = doArgs(arguments, ['string', 'string|number', ['object', {}], 'function']);

      var _doArgs16 = _slicedToArray(_doArgs15, 4);

      jobName = _doArgs16[0];
      buildNumber = _doArgs16[1];
      customParams = _doArgs16[2];
      callback = _doArgs16[3];


      doRequest({
        urlPattern: [BUILD_INFO, jobName, buildNumber]
      }, customParams, callback);
    }
171985
def job_data(cls, job_id=dummy_job_id, branches=None, **kwargs):
        
        return dict(job_id=job_id, branches=branches or [])
370049
def jobs(self):
        
        jobs = list()
        for data in self.data.jobs:
            jobs.append(JobInstance(session=self._session, data=data, stage=self))

        return jobs
869546
def add_functions(spec_dict: Mapping[str, Any]) -> Mapping[str, Any]:
    

    # Class 'Mapping' does not define '__setitem__', so the '[]' operator cannot be used on its instances
    spec_dict["functions"]["list"] = []
    spec_dict["functions"]["list_long"] = []
    spec_dict["functions"]["list_short"] = []

    spec_dict["functions"]["primary"] = {}
    spec_dict["functions"]["primary"]["list_long"] = []
    spec_dict["functions"]["primary"]["list_short"] = []

    spec_dict["functions"]["modifier"] = {}
    spec_dict["functions"]["modifier"]["list_long"] = []
    spec_dict["functions"]["modifier"]["list_short"] = []

    spec_dict["functions"]["to_short"] = {}
    spec_dict["functions"]["to_long"] = {}

    for func_name in spec_dict["functions"]["info"]:

        abbreviated_name = spec_dict["functions"]["info"][func_name]["abbreviation"]

        spec_dict["functions"]["list"].extend((func_name, abbreviated_name))

        spec_dict["functions"]["list_long"].append(func_name)
        spec_dict["functions"]["list_short"].append(abbreviated_name)

        if spec_dict["functions"]["info"][func_name]["type"] == "primary":
            spec_dict["functions"]["primary"]["list_long"].append(func_name)
            spec_dict["functions"]["primary"]["list_short"].append(abbreviated_name)
        else:
            spec_dict["functions"]["modifier"]["list_long"].append(func_name)
            spec_dict["functions"]["modifier"]["list_short"].append(abbreviated_name)

        spec_dict["functions"]["to_short"][abbreviated_name] = abbreviated_name
        spec_dict["functions"]["to_short"][func_name] = abbreviated_name

        spec_dict["functions"]["to_long"][abbreviated_name] = func_name
        spec_dict["functions"]["to_long"][func_name] = func_name

    return spec_dict
862782
def sg_producer_func(func):
    
    """
    @wraps(func)
    def wrapper(**kwargs):
        r"""Manages arguments of `tf.sg_opt`.

        Args:
          **kwargs:
            source: A source queue list to enqueue
            dtypes: Input data types of each tensor
            out_dtypes: Output data types of each tensor ( If None, same as dtypes )
            capacity: Queue capacity. Default is 32.
            num_threads: Number of threads. Default is 1.
        """

        # default option
        opt = tf.sg_opt(kwargs) + tf.sg_opt(dtypes=[tf.sg_floatx], capacity=32, num_threads=1)

        # source queue list check
        assert opt.source is not None, 'source is mandatory.'
        if type(opt.source) is not list and type(opt.source) is not tuple:
            opt.source = [opt.source]
        if type(opt.dtypes) is not list and type(opt.dtypes) is not tuple:
            opt.dtypes = [opt.dtypes]
        # default out_dtypes
        if opt.out_dtypes is None:
            opt.out_dtypes = opt.dtypes
        if type(opt.out_dtypes) is not list and type(opt.out_dtypes) is not tuple:
            opt.out_dtypes = [opt.out_dtypes]
        assert len(opt.source) == len(opt.dtypes), 'Source and dtypes should have same length.'

        # enqueue function
        def enqueue_func(sess, op):
            # read data from source queue
            data = func(sess.run(opt.source))
            # create feeder dict
            feed_dict = {}
            for ph, col in zip(placeholders, data):
                feed_dict[ph] = col
            # run session
            sess.run(op, feed_dict=feed_dict)

        # create place holder list
        placeholders = []
        for dtype in opt.dtypes:
            placeholders.append(tf.placeholder(dtype=dtype))

        # create FIFO queue
        queue = tf.FIFOQueue(opt.capacity, dtypes=opt.out_dtypes)

        # enqueue operation
        enqueue_op = queue.enqueue(placeholders)

        # create queue runner
        runner = _FuncQueueRunner(enqueue_func, queue, [enqueue_op] * opt.num_threads)

        # register to global collection
        tf.train.add_queue_runner(runner)

        # return de-queue operation
        return queue.dequeue()

    return wrapper
1366541
def _create_job_info(self, job_dir):
        
        meta = self._build_job_meta(job_dir)

        self.logger.debug("Create job: %s" % meta)

        job_record = JobRecord.from_json(meta)
        job_record.save()
1248799
def spawn_batch_jobs(job, shared_ids, input_args):
    
    samples = []
    config = input_args['config']
    with open(config, 'r') as f_in:
        for line in f_in:
            line = line.strip().split(',')
            uuid = line[0]
            urls = line[1:]
            samples.append((uuid, urls))
    for sample in samples:
        job.addChildJobFn(alignment, shared_ids, input_args, sample, cores=32, memory='20 G', disk='100 G')
1312606
def _build_job_dict(self):
        """"""
        if self.args['dry_run']:
            status = JobStatus.unknown
        else:
            status = JobStatus.not_ready

        base_config = self.scatter_link.args

        for jobkey, job_config in sorted(self._job_configs.items()):
            full_job_config = base_config.copy()
            full_job_config.update(job_config)
            ScatterGather._make_scatter_logfile_name(jobkey,
                                                     self.linkname, full_job_config)
            logfile = job_config.get('logfile')
            self._scatter_link._register_job(key=jobkey,
                                             job_config=full_job_config,
                                             logfile=logfile,
                                             status=status)
439413
def add_job(self, id, func, **kwargs):
        

        job_def = dict(kwargs)
        job_def['id'] = id
        job_def['func'] = func
        job_def['name'] = job_def.get('name') or id

        fix_job_def(job_def)

        return self._scheduler.add_job(**job_def)