Appended a new job to this JobInfo node.
        :param job: The new job to append.
        :param bool merged: Whether it is a merged job or not.
        :param bool widened: Whether it is a widened job or not.
979225
def add_job(self, job, merged=False, widened=False):
        

        job_type = ''
        if merged:
            job_type = 'merged'
        elif widened:
            job_type = 'widened'
        self.jobs.append((job, job_type))
631398
private void appendToQueueInternal (final ScheduledJob job) {
		jobIdQueue.add(job.jobId());
		queuedJobs.put(job.jobId(), job);
	}
149243
@Override
  public void addJob(final Job job) throws JobExistsException {
    log.info("adding job: {}", job);
    final JobId id = job.getId();
    final UUID operationId = UUID.randomUUID();
    final String creationPath = Paths.configJobCreation(id, operationId);
    final ZooKeeperClient client = provider.get("addJob");
    try {
      try {
        client.ensurePath(Paths.historyJob(id));
        client.transaction(create(Paths.configJob(id), job),
            create(Paths.configJobRefShort(id), id),
            create(Paths.configJobHosts(id)),
            create(creationPath),
            // Touch the jobs root node so that its version is bumped on every job
            // change down the tree. Effectively, make it that version == cVersion.
            set(Paths.configJobs(), UUID.randomUUID().toString().getBytes()));
      } catch (final NodeExistsException e) {
        if (client.exists(creationPath) != null) {
          // The job was created, we're done here
          return;
        }
        throw new JobExistsException(id.toString());
      }
    } catch (NoNodeException e) {
      throw new HeliosRuntimeException("adding job " + job + " failed due to missing ZK path: "
                                       + e.getPath(), e);
    } catch (final KeeperException e) {
      throw new HeliosRuntimeException("adding job " + job + " failed", e);
    }
  }
62864
private void makeJobRunning(JobInProgress job, JobSchedulingInfo oldInfo, 
                              QueueInfo qi) {
    // Removing of the job from job list is responsibility of the
    //initialization poller.
    // Add the job to the running queue
    qi.addRunningJob(job);
  }
303068
def add_job_to_db(self, key, job):
        """"""
        job_msg = self.registry.deep_encode(job)
        prov = prov_key(job_msg)

        def set_link(duplicate_id):
            self.cur.execute(
                'update "jobs" set "link" = ?, "status" = ? where "id" = ?',
                (duplicate_id, Status.DUPLICATE, key))

        with self.lock:
            self.cur.execute(
                'select * from "jobs" where "prov" = ? '
                'and (("result" is not null) or '
                '("session" = ? and "link" is null))',
                (prov, self.session))
            rec = self.cur.fetchone()
            rec = JobEntry(*rec) if rec is not None else None

            self.cur.execute(
                'update "jobs" set "prov" = ?, "version" = ?, "function" = ?, '
                '"arguments" = ?, "status" = ? where "id" = ?',
                (prov, job_msg['data']['hints'].get('version'),
                 json.dumps(job_msg['data']['function']),
                 json.dumps(job_msg['data']['arguments']),
                 Status.WAITING,
                 key))

            if not rec:
                # no duplicate found, go on
                return 'initialized', None

            set_link(rec.id)

            if rec.result is not None and rec.status == Status.WORKFLOW:
                # the found duplicate returned a workflow
                if rec.link is not None:
                    # link is set, so result is fully realized
                    self.cur.execute(
                        'select * from "jobs" where "id" = ?',
                        (rec.link,))
                    rec = self.cur.fetchone()
                    assert rec is not None, "database integrity violation"
                    rec = JobEntry(*rec)

                else:
                    # link is not set, the result is still waited upon
                    assert rec.session == self.session, \
                        "database integrity violation"
                    self.attached[rec.id].append(key)
                    return 'attached', None

            if rec.result is not None:
                # result is found! return it
                result_value = self.registry.from_json(rec.result, deref=True)
                result = ResultMessage(
                    key, 'retrieved', result_value, None)
                return 'retrieved', result

            if rec.session == self.session:
                # still waiting for result, attach
                self.attached[rec.id].append(key)
                return 'attached', None
12139
def perform_job(self, job):
        

        if self.config["trace_memory"]:
            job.trace_memory_start()

        set_current_job(job)

        try:
            job.perform()

        except MaxConcurrencyInterrupt:
            self.log.error("Max concurrency reached")
            job._save_status("maxconcurrency", exception=True)

        except RetryInterrupt:
            self.log.error("Caught retry")
            job.save_retry(sys.exc_info()[1])

        except MaxRetriesInterrupt:
            self.log.error("Max retries reached")
            job._save_status("maxretries", exception=True)

        except AbortInterrupt:
            self.log.error("Caught abort")
            job.save_abort()

        except TimeoutInterrupt:
            self.log.error("Job timeouted after %s seconds" % job.timeout)
            job._save_status("timeout", exception=True)

        except JobInterrupt:
            self.log.error("Job interrupted")
            job._save_status("interrupt", exception=True)

        except Exception:
            self.log.error("Job failed")
            job._save_status("failed", exception=True)

        finally:

            set_current_job(None)

            self.done_jobs += 1

            if self.config["trace_memory"]:
                job.trace_memory_stop()
439413
def add_job(self, id, func, **kwargs):
        

        job_def = dict(kwargs)
        job_def['id'] = id
        job_def['func'] = func
        job_def['name'] = job_def.get('name') or id

        fix_job_def(job_def)

        return self._scheduler.add_job(**job_def)
775568
def _checkJobGraphAcylicDFS(self, stack, visited, extraEdges):
        
        if self not in visited:
            visited.add(self)
            stack.append(self)
            for successor in self._children + self._followOns + extraEdges[self]:
                successor._checkJobGraphAcylicDFS(stack, visited, extraEdges)
            assert stack.pop() == self
        if self in stack:
            stack.append(self)
            raise JobGraphDeadlockException("A cycle of job dependencies has been detected '%s'" % stack)
150782
def add_jobs(jobs)
      raise PoolClosed.new("Pool #{self} is currently closed") if pool_closed?
      raise ArgumentError.new("Expecting an array but received: #{jobs.class}") unless jobs.is_a?(Array)
      @queue.pause
      begin
        jobs.each do |job|
          case job
          when Proc
            @queue << [job, []]
          when Array
            raise ArgumentError.new('Jobs to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]') unless job.size == 2 and job[0].is_a?(Proc) and job[1].is_a?(Array)
            @queue << [job.shift, job]
          else
            raise ArgumentError.new('Jobs to be processed by the pool must be a proc/lambda or [proc/lambda, [*args]]')
          end
        end
      ensure
        num = jobs.size - @threads.select{|t|t.waiting?}.size
        @lock.synchronize{ num.times{ create_thread(:nowait) } if num > 0 }
        @queue.unpause
      end
      true
    end
974968
def merged():
    # type: () -> None
    
    develop = conf.get('git.devel_branch', 'develop')
    master = conf.get('git.master_branch', 'master')
    branch = git.current_branch(refresh=True)

    common.assert_branch_type('hotfix')

    # Pull master with the merged hotfix
    common.git_checkout(master)
    common.git_pull(master)

    # Merge to develop
    common.git_checkout(develop)
    common.git_pull(develop)
    common.git_merge(develop, branch.name)

    # Cleanup
    common.git_branch_delete(branch.name)
    common.git_prune()

    common.git_checkout(master)
524355
async def new_job(self, message: BackendNewJob):
        
        self._logger.info("Received request for jobid %s", message.job_id)
        future_results = asyncio.Future()
        out = await self._loop.run_in_executor(None, lambda: self.__new_job_sync(message, future_results))
        self._create_safe_task(self.handle_running_container(**out, future_results=future_results))
        await self._timeout_watcher.register_container(out["container_id"], out["orig_time_limit"], out["orig_hard_time_limit"])
1237844
def job_success(self, job, queue, job_result):
        

        # display what was done
        obj = job.get_object()
        message = '[%s|%s] %s [%s]' % (queue.name.hget(),
                                       obj.pk.get(),
                                       job_result,
                                       threading.current_thread().name)
        self.log(message)

        # default stuff: update job and queue statuses, and do logging
        super(FullNameWorker, self).job_success(job, queue, job_result)

        # update the queue's dates
        queue_fields_to_update = {
            'last_job_date': str(datetime.utcnow())
        }
        if not queue.first_job_date.hget():
            queue_fields_to_update['first_job_date'] = queue_fields_to_update['last_job_date']
        queue.hmset(**queue_fields_to_update)

        # update the jobs counter on the queue
        queue.jobs_counter.hincrby(1)

        # save a ref to the job to display at the end of this example script
        self.jobs.append(int(job.pk.get()))
828231
def new_job(frontier, job_conf):
    ''''''
    validate_conf(job_conf)
    job = Job(frontier.rr, {
                "conf": job_conf, "status": "ACTIVE",
                "started": doublethink.utcnow()})
    if "id" in job_conf:
        job.id = job_conf["id"]
    if "max_claimed_sites" in job_conf:
        job.max_claimed_sites = job_conf["max_claimed_sites"]
    job.save()

    sites = []
    pages = []
    for seed_conf in job_conf["seeds"]:
        merged_conf = merge(seed_conf, job_conf)
        merged_conf.pop("seeds")
        merged_conf["job_id"] = job.id
        merged_conf["seed"] = merged_conf.pop("url")
        site = brozzler.Site(frontier.rr, merged_conf)
        site.id = str(uuid.uuid4())
        sites.append(site)
        pages.append(new_seed_page(frontier, site))

    # insert in batches to avoid this error
    # rethinkdb.errors.ReqlDriverError: Query size (167883036) greater than maximum (134217727) in:
    for batch in (pages[i:i+500] for i in range(0, len(pages), 500)):
        logging.info('inserting batch of %s pages', len(batch))
        result = frontier.rr.table('pages').insert(batch).run()
    for batch in (sites[i:i+100]  for i in range(0, len(sites), 100)):
        logging.info('inserting batch of %s sites', len(batch))
        result = frontier.rr.table('sites').insert(batch).run()
    logging.info('job %s fully started', job.id)

    return job
1301970
def add_result(self, job):
        
        self.cur.execute(
            "INSERT INTO history VALUES(?,?,?,?)",
            (job["id"], job["description"], job["last-run"], job["last-run-result"]))

        return True
34757
def add_job_from_json(self, job_json, destructive=False):
        
        logger.debug('Importing job from JSON document: {0}'.format(job_json))
        rec = self.backend.decode_import_json(job_json)
        if destructive:
            try:
                self.delete_job(rec['name'])
            except DagobahError:  # expected if no job with this name
                pass
        self._add_job_from_spec(rec, use_job_id=False)

        self.commit(cascade=True)
682042
def submit_jobs(self, link, job_dict=None, job_archive=None, stream=sys.stdout):
        
        if link is None:
            return JobStatus.no_job
        if job_dict is None:
            job_keys = link.jobs.keys()
        else:
            job_keys = sorted(job_dict.keys())

        # copy & reverse the keys b/c we will be popping item off the back of
        # the list
        unsubmitted_jobs = job_keys
        unsubmitted_jobs.reverse()

        failed = False
        if unsubmitted_jobs:
            if stream != sys.stdout:
                sys.stdout.write('Submitting jobs (%i): ' %
                                 len(unsubmitted_jobs))
                sys.stdout.flush()
        while unsubmitted_jobs:
            status = get_lsf_status()
            njob_to_submit = min(self._max_jobs - status['NJOB'],
                                 self._jobs_per_cycle,
                                 len(unsubmitted_jobs))

            if self._dry_run:
                njob_to_submit = len(unsubmitted_jobs)

            for i in range(njob_to_submit):
                job_key = unsubmitted_jobs.pop()

                # job_details = job_dict[job_key]
                job_details = link.jobs[job_key]
                job_config = job_details.job_config
                if job_details.status == JobStatus.failed:
                    clean_job(job_details.logfile, {}, self._dry_run)
                    # clean_job(job_details.logfile,
                    #          job_details.outfiles, self.args['dry_run'])

                job_config['logfile'] = job_details.logfile
                new_job_details = self.dispatch_job(
                    link, job_key, job_archive, stream)
                if new_job_details.status == JobStatus.failed:
                    failed = True
                    clean_job(new_job_details.logfile,
                              new_job_details.outfiles, self._dry_run)
                link.jobs[job_key] = new_job_details

            if unsubmitted_jobs:
                if stream != sys.stdout:
                    sys.stdout.write('.')
                    sys.stdout.flush()
                stream.write('Sleeping %.0f seconds between submission cycles\n' %
                             self._time_per_cycle)
                time.sleep(self._time_per_cycle)

        if failed:
            return JobStatus.failed

        if stream != sys.stdout:
            sys.stdout.write('!\n')

        return JobStatus.done
556599
def _merge_asized(base, other, level=0):
    
    ref2key = lambda ref: ref.name.split(':')[0]
    base.size += other.size
    base.flat += other.flat
    if level > 0:
        base.name = ref2key(base)
    # Add refs from other to base. Any new refs are appended.
    base.refs = list(base.refs) # we may need to append items
    refs = {}
    for ref in base.refs:
        refs[ref2key(ref)] = ref
    for ref in other.refs:
        key = ref2key(ref)
        if key in refs:
            _merge_asized(refs[key], ref, level=level+1)
        else:
            # Don't modify existing Asized instances => deepcopy
            base.refs.append(deepcopy(ref))
            base.refs[-1].name = key