Submit tasks by interpolating a shell script defined in job_template
481674
def _submit_task_with_template(self, task_ids):
        ''''''
        runtime = self.config
        runtime.update({
            'workdir': os.getcwd(),
            'cur_dir': os.getcwd(),  # for backward compatibility
            'verbosity': env.verbosity,
            'sig_mode': env.config.get('sig_mode', 'default'),
            'run_mode': env.config.get('run_mode', 'run'),
            'home_dir': os.path.expanduser('~')
        })
        if '_runtime' in env.sos_dict:
            runtime.update({
                x: env.sos_dict['_runtime'][x]
                for x in ('nodes', 'cores', 'workdir', 'mem', 'walltime')
                if x in env.sos_dict['_runtime']
            })
        if 'nodes' not in runtime:
            runtime['nodes'] = 1
        if 'cores' not in runtime:
            runtime['cores'] = 1

        # let us first prepare a task file
        job_text = ''
        for task_id in task_ids:
            runtime['task'] = task_id
            try:
                job_text += cfg_interpolate(self.job_template, runtime)
                job_text += '\n'
            except Exception as e:
                raise ValueError(
                    f'Failed to generate job file for task {task_id}: {e}')

        filename = task_ids[0] + ('.sh' if len(task_ids) == 1 else
                                  f'-{task_ids[-1]}.sh')
        # now we need to write a job file
        job_file = os.path.join(
            os.path.expanduser('~'), '.sos', 'tasks', filename)
        # do not translate newline under windows because the script will be executed
        # under linux/mac
        with open(job_file, 'w', newline='') as job:
            job.write(job_text)

        # then copy the job file to remote host if necessary
        self.agent.send_task_file(job_file)

        try:
            cmd = f'bash ~/.sos/tasks/{filename}'
            self.agent.run_command(cmd, wait_for_task=self.wait_for_task)
        except Exception as e:
            raise RuntimeError(f'Failed to submit task {task_ids}: {e}')
        return True
51331
def submit(self, command_line, name = None, array = None, dependencies = [], exec_dir = None, log_dir = None, dry_run = False, stop_on_failure = False, **kwargs):
    """"""
    # remove duplicate dependencies
    dependencies = sorted(list(set(dependencies)))

    # add job to database
    self.lock()
    job = add_job(self.session, command_line=command_line, name=name, dependencies=dependencies, array=array, exec_dir=exec_dir, log_dir=log_dir, stop_on_failure=stop_on_failure)
    logger.info("Added job '%s' to the database", job)

    if dry_run:
      print("Would have added the Job", job, "to the database to be executed locally.")
      self.session.delete(job)
      logger.info("Deleted job '%s' from the database due to dry-run option", job)
      job_id = None
    else:
      job_id = job.unique

    # return the new job id
    self.unlock()
    return job_id
488513
def process_templates(input_dir, target_dir, context):
    
    if not target_dir.endswith('/'):
        target_dir = "{}/".format(target_dir)
    if not os.path.exists(target_dir):
        LOGGER.info('Creating: {}'.format(target_dir))
        os.makedirs(target_dir)
    replicate_directory_tree(input_dir, target_dir)
    templates = find_templates(input_dir)
    for templ in templates:
        output_file = templ.replace(input_dir, target_dir)
        output_file = output_file[:-len('.mustache')]
        render_template(templ,  output_file, context)
1147585
def submit_job(self, job_definition, parameters, job_name=None, queue=None):
        """"""
        if job_name is None:
            job_name = _random_id()
        response = self._client.submit_job(
            jobName=job_name,
            jobQueue=queue or self.get_active_queue(),
            jobDefinition=job_definition,
            parameters=parameters
        )
        return response['jobId']
735322
def rake(task,*arguments)
      remote_shell do |shell|
        shell.cd(shell.uri.path)
        shell.rake(task,*arguments)
      end

      return true
    end
1341306
def add(self, command_template, job_class):
        
        job = JobTemplate(command_template.alias,
            command_template=command_template,
            depends_on=command_template.depends_on, queue=self.queue,
            job_class=job_class)
        self.queue.push(job)
1234675
def submit(self, job):
        
        self._check_job(job)

        if job.workingdir is None:
            job.workingdir = self.default_wdir
        job.imageid = du.create_provisioned_image(self.client, job.image,
                                                  job.workingdir, job.inputs)

        container_args = self._generate_container_args(job)

        job.rundata.container = self.client.create_container(job.imageid, **container_args)
        self.client.start(job.rundata.container)
        job.rundata.containerid = job.rundata.container['Id']
        job.jobid = job.rundata.containerid
625918
def submit(script, workspace, **params):
    """"""
    from klab import cluster, process

    # Make sure the rosetta symlink has been created.

    if not os.path.exists(workspace.rosetta_dir):
        raise pipeline.RosettaNotFound(workspace)

    # Parse some job parameters for the keyword arguments.

    params = dict((k, v) for k, v in params.items() if v is not None)
    test_run = params.get('test_run', False)
    nstruct = params.get('nstruct')
    max_runtime = params.get('max_runtime', '6:00:00')
    max_memory = params.get('max_memory', '1G')

    if test_run:
        nstruct = 1

    if nstruct is None:
        raise TypeError("submit() requires the keyword argument 'nstruct' for production runs.")

    # Use Jinja to render the XML script that will be passed to rosetta.

    finalize_protocol(workspace, params)

    # Submit the job and put it immediately into the hold state.

    qsub_command = 'qsub', '-h', '-cwd',
    qsub_command += '-o', workspace.log_dir,
    qsub_command += '-j', 'y',
    qsub_command += '-t', '1-{0}'.format(nstruct),
    qsub_command += '-l', 'h_rt={0}'.format(max_runtime),
    qsub_command += '-l', 'mem_free={0}'.format(max_memory),
    qsub_command += pipeline.big_job_path(script),
    qsub_command += workspace.focus_dir,

    status = process.check_output(qsub_command)
    status_pattern = re.compile(r'Your job-array (\d+).[0-9:-]+ \(".*"\) has been submitted')
    status_match = status_pattern.match(status)

    if not status_match:
        print status
        sys.exit()

    # Figure out the job id, then make a params file specifically for it.

    job_id = status_match.group(1)

    with open(workspace.job_info_path(job_id), 'w') as file:
        json.dump(params, file)

    # Release the hold on the job.

    qrls_command = 'qrls', job_id
    process.check_output(qrls_command)
    print status,
800119
def run_script_with_context(script_path, cwd, context):
    
    _, extension = os.path.splitext(script_path)

    contents = io.open(script_path, 'r', encoding='utf-8').read()

    with tempfile.NamedTemporaryFile(
        delete=False,
        mode='wb',
        suffix=extension
    ) as temp:
        env = StrictEnvironment(
            context=context,
            keep_trailing_newline=True,
        )
        template = env.from_string(contents)
        output = template.render(**context)
        temp.write(output.encode('utf-8'))

    run_script(temp.name, cwd)
1230931
def build_job_scripts(root_dir, jobs):
    
    # Loop over the job list, creating each job script in turn, and then adding
    # scriptPath to the Job object
    for job in jobs:
        scriptpath = os.path.join(root_dir, "jobs", job.name)
        with open(scriptpath, "w") as scriptfile:
            scriptfile.write("#!/bin/sh\n#$ -S /bin/bash\n%s\n" % job.script)
        job.scriptpath = scriptpath
666895
def wrap_worker__run(*args, **kwargs):
    
    # Ignore parent's attempts to murder us when we still need to write
    # profiling output.
    if mitogen.core._profile_hook.__name__ != '_profile_hook':
        signal.signal(signal.SIGTERM, signal.SIG_IGN)

    ansible_mitogen.logging.set_process_name('task')
    ansible_mitogen.affinity.policy.assign_worker()
    return mitogen.core._profile_hook('WorkerProcess',
        lambda: worker__run(*args, **kwargs)
    )
152170
def submit_comp_job(self, path, callback, method, *args, **kwargs):
        """"""
        if self.executor is None:
            with self.handling_exceptions():
                callback(getattr(self.comp, method)(*args, **kwargs))
        else:
            path = showpath(path)
            with logger.in_path(path):  # pickle the compiler in the path context
                future = self.executor.submit(multiprocess_wrapper(self.comp, method), *args, **kwargs)

            def callback_wrapper(completed_future):
                """Ensures that all errors are always caught, since errors raised in a callback won't be propagated."""
                with logger.in_path(path):  # handle errors in the path context
                    with self.handling_exceptions():
                        result = completed_future.result()
                        callback(result)
            future.add_done_callback(callback_wrapper)
230527
def script_template(state, host, template_filename, chdir=None, **data):
    '''
    
    '''

    temp_file = state.get_temp_filename(template_filename)
    yield files.template(state, host, template_filename, temp_file, **data)

    yield chmod(temp_file, '+x')

    if chdir:
        yield 'cd {0} && {1}'.format(chdir, temp_file)
    else:
        yield temp_file
916310
def update_payload(self, fields=None):
        """"""
        payload = super(JobTemplate, self).update_payload(fields)
        effective_user = payload.pop(u'effective_user', None)
        if effective_user:
            payload[u'ssh'] = {u'effective_user': effective_user}
        return {u'job_template': payload}
630052
def submitTemplate(id, data={}):
        
        conn = Qubole.agent()
        path = str(id) + "/run"
        return conn.post(Template.element_path(path), data)
990658
function (jobName, customParams, callback) {
      [jobName, customParams, callback] = doArgs(arguments, ['string', ['object', {}], 'function']);

      doRequest({
        urlPattern: [LAST_BUILD, jobName]
      }, customParams, callback);
    }
521590
function pluck(path, cb) {
  var tasks = [];
  fs.readFile(path, 'utf8', function (err, str) {
    if (err) return cb(err);

    /**
     * Skip binary-files
     */

    if (isBinary(str)) return;

    /**
     * Pointer within the file
     */

    var lines = str.split('\n'),
        linum = 0;

    while (lines.length) {
      var line = lines.shift(); linum++;

      /**
       * Check for task-comment
       */

      if (!pattern.test(line)) continue;

      /**
       * Old-style comments
       * @see http://www.riedquat.de/TR/trunk/TODO_Syntax/
       */

      var matches = line.match(isoPattern);
      if (matches != null) {
        matches = matches.toString().split(':');
        var labels = ['#'+matches[0].toLowerCase()],
            added = new Date(matches[1]),
            assignee = '@'+matches[2].replace(/[\\s]*/, '-'),
            linestart = linum,
            title = matches[3],
            body = '';
      }

      /**
       * New-style comments
       *
       * TODO Write a comment specification
       *    | Should be located at the address below
       *
       * TODO Share patterns between modules
       *    | Perhaps even make all of them global
       *
       * @see http://hkjels.github.com/ntask/syntax.html
       */

      else {
        var added = new Date(),
            assignee = (line.match(/@[\w\[\d\]-_]*/g)||['@none'])[0],
            title = line.replace(new RegExp('.*('+KEYWORDS+')'), '')
              .replace(assignee, '')
              .replace(/[^a-z0-9\!\]\)]*$/i, '').trim(),
            linestart = linum,
            body = (function body() {
              var next = lines.shift();
              if (/[\\s]*[\\\*\%\-(" )"]{0,3}[\|]/g.test(next)) {
                linum++;
                return next.split('|').slice(1).join('').trim()+'\n'+body();
              }
              else {
                lines.unshift(next);
                return '';
              }
            })(),
            labels = ((title+body).match(/#[\w]*[\[\]a-f0-9\.]*/gi) || [])
              .map(lowertrim).filter(minlength),
            keyword = line.match(new RegExp('.*('+KEYWORDS+')'))[1].toLowerCase();
            labels.unshift('#'+keyword);
      }
      var done = labels.indexOf('#done') != -1;

      /**
       * Parsed task-object ready for model-validation
       */

      var task = {
          added: added
        , assignee: assignee
        , body: body
        , done: done
        , file: path
        , labels: labels
        , line: linestart
        , title: title
      };
      tasks.push(task);
    }
    cb(false, tasks);
  });
}