Check each issued job - if it is running for longer than desirable
        issue a kill instruction.
        Wait for the job to die then we pass the job to processFinishedJob.
181761
def reissueOverLongJobs(self):
        
        maxJobDuration = self.config.maxJobDuration
        jobsToKill = []
        if maxJobDuration < 10000000:  # We won't bother doing anything if rescue time > 16 weeks.
            runningJobs = self.batchSystem.getRunningBatchJobIDs()
            for jobBatchSystemID in list(runningJobs.keys()):
                if runningJobs[jobBatchSystemID] > maxJobDuration:
                    logger.warn("The job: %s has been running for: %s seconds, more than the "
                                "max job duration: %s, we'll kill it",
                                str(self.jobBatchSystemIDToIssuedJob[jobBatchSystemID].jobStoreID),
                                str(runningJobs[jobBatchSystemID]),
                                str(maxJobDuration))
                    jobsToKill.append(jobBatchSystemID)
            self.killJobs(jobsToKill)
702528
def wait_for_job_to_finish(
            self,
            job_id,
            sec_to_sleep=5.0,
            max_retries=100000):
        

        not_done = True
        retry_attempt = 1
        while not_done:

            if self.debug:
                log.info(("JOBSTATUS getting job.id={} details")
                         .format(
                            job_id))

            response = self.get_job_by_id(job_id)

            if self.debug:
                log.info(("JOBSTATUS got job.id={} response={}")
                         .format(
                            job_id,
                            response))

            if response["status"] != SUCCESS:
                log.error(("JOBSTATUS failed to get job.id={} with error={}")
                          .format(
                            job_id,
                            response["error"]))
                return self.build_response(
                    status=ERROR,
                    error=response["error"],
                    data=response["data"])
            # stop if this failed getting the job details

            job_data = response.get(
                "data",
                None)

            if not job_data:
                return self.build_response(
                    status=ERROR,
                    error="failed to find job dictionary in response",
                    data=response["data"])

            job_status = job_data["status"]

            if job_status == "finished" \
               or job_status == "completed" \
               or job_status == "launched":

                if self.debug:
                    log.info(("job.id={} is done with status={}")
                             .format(
                                job_id,
                                job_status))

                result_id = job_data["predict_manifest"]["result_id"]

                if self.debug:
                    log.info(("JOBRESULT getting result.id={} details")
                             .format(
                                result_id))

                response = self.get_result_by_id(result_id)

                if self.debug:
                    log.info(("JOBRESULT got result.id={} response={}")
                             .format(
                                result_id,
                                response))

                if response["status"] != SUCCESS:
                    log.error(("JOBRESULT failed to get "
                               "result.id={} with error={}")
                              .format(
                                result_id,
                                response["error"]))
                    return self.build_response(
                        status=ERROR,
                        error=response["error"],
                        data=response["data"])
                # stop if this failed getting the result details

                result_data = response.get(
                    "data",
                    None)

                if result_data["status"] == "finished":
                    full_response = {
                        "job": job_data,
                        "result": result_data
                    }
                    not_done = False
                    return self.build_response(
                        status=SUCCESS,
                        error="",
                        data=full_response)
                else:
                    if retry_attempt % 100 == 0:
                        if self.verbose:
                            log.info(("result_id={} are not done retry={}")
                                     .format(
                                        result_id,
                                        retry_attempt))

                    retry_attempt += 1
                    if retry_attempt > max_retries:
                        err_msg = ("failed waiting "
                                   "for job.id={} result.id={} "
                                   "to finish").format(
                                    job_id,
                                    result_id)
                        log.error(err_msg)
                        return self.build_response(
                            status=ERROR,
                            error=err_msg)
                    else:
                        time.sleep(sec_to_sleep)
                    # wait while results are written to the db
            else:

                retry_attempt += 1
                if retry_attempt > max_retries:
                    err_msg = ("failed waiting "
                               "for job.id={} to finish").format(
                                   job_id)
                    log.error(err_msg)
                    return self.build_response(
                        status=ERROR,
                        error=err_msg)
                else:
                    if self.verbose:
                        if retry_attempt % 100 == 0:
                            log.info(("waiting on job.id={} retry={}")
                                     .format(
                                        job_id,
                                        retry_attempt))
                    # if logging just to show this is running
                    time.sleep(sec_to_sleep)
9148
boolean detectLongRunningJob(long currentTime, Job job) {
		if(job.status() == JobStatus.RUNNING && !longRunningJobs.containsKey(job)) {
			int jobExecutionsCount = job.executionsCount();
			Long jobStartedtimeInMillis = job.lastExecutionStartedTimeInMillis();
			Thread threadRunningJob = job.threadRunningJob();

			if(jobStartedtimeInMillis != null
				&& threadRunningJob != null
				&& currentTime - jobStartedtimeInMillis > detectionThresholdInMillis) {
				logger.warn(
					"Job '{}' is still running after {}ms (detection threshold = {}ms), stack trace = {}",
					job.name(),
					currentTime - jobStartedtimeInMillis,
					detectionThresholdInMillis,
					Stream
						.of(threadRunningJob.getStackTrace())
						.map(StackTraceElement::toString)
						.collect(Collectors.joining("\n  "))
				);

				longRunningJobs.put(
					job,
					new LongRunningJobInfo(jobStartedtimeInMillis, jobExecutionsCount)
				);

				return true;
			}
		}
		return false;
	}
1055393
@Override
	public void run() {

		final long currentTime = System.currentTimeMillis();

		synchronized (this.collectedEvents) {

			final Iterator<JobID> it = this.collectedEvents.keySet().iterator();
			while (it.hasNext()) {

				final JobID jobID = it.next();
				final List<AbstractEvent> eventList = this.collectedEvents.get(jobID);
				if (eventList == null) {
					continue;
				}

				final Iterator<AbstractEvent> it2 = eventList.iterator();
				while (it2.hasNext()) {

					final AbstractEvent event = it2.next();
					// If the event is older than TIMERTASKINTERVAL, remove it
					if ((event.getTimestamp() + this.timerTaskInterval) < currentTime) {
						archiveEvent(jobID, event);
						it2.remove();
					}
				}

				if (eventList.isEmpty()) {
					it.remove();
				}
			}
		}

		synchronized (this.recentJobs) {

			final Iterator<Map.Entry<JobID, RecentJobEvent>> it = this.recentJobs.entrySet().iterator();
			while (it.hasNext()) {

				final Map.Entry<JobID, RecentJobEvent> entry = it.next();
				final JobStatus jobStatus = entry.getValue().getJobStatus();

				// Only remove jobs from the list which have stopped running
				if (jobStatus != JobStatus.FINISHED && jobStatus != JobStatus.CANCELED
					&& jobStatus != JobStatus.FAILED) {
					continue;
				}

				// Check time stamp of last job status update
				if ((entry.getValue().getTimestamp() + this.timerTaskInterval) < currentTime) {
					archiveJobevent(entry.getKey(), entry.getValue());
					it.remove();
					synchronized (this.recentManagementGraphs) {
						archiveManagementGraph(entry.getKey(), this.recentManagementGraphs.get(entry.getKey()));
						this.recentManagementGraphs.remove(entry.getValue());
					}
					synchronized (this.recentNetworkTopologies) {
						archiveNetworkTopology(entry.getKey(), this.recentNetworkTopologies.get(entry.getKey()));
						this.recentNetworkTopologies.remove(entry.getValue());
					}
				}
			}
		}
	}
661986
synchronized void finalizeJob(JobInProgress job) {
    // Mark the 'non-running' tasks for pruning
    markCompletedJob(job);

    JobEndNotifier.registerNotification(job.getJobConf(), job.getStatus());

    // start the merge of log files
    JobID id = job.getStatus().getJobID();

    // mark the job as completed
    try {
      JobHistory.JobInfo.markCompleted(id);
    } catch (IOException ioe) {
      LOG.info("Failed to mark job " + id + " as completed!", ioe);
    }

    final JobTrackerInstrumentation metrics = getInstrumentation();
    metrics.finalizeJob(conf, id);

    long now = getClock().getTime();

    // mark the job for cleanup at all the trackers
    addJobForCleanup(id);

    try {
      File userFileForJob =
        new File(lDirAlloc.getLocalPathToRead(SUBDIR + "/" + id,
                                              conf).toString());
      if (userFileForJob != null) {
        userFileForJob.delete();
      }
    } catch (IOException ioe) {
      LOG.info("Failed to delete job id mapping for job " + id, ioe);
    }

    // add the blacklisted trackers to potentially faulty list
    if (job.getStatus().getRunState() == JobStatus.SUCCEEDED) {
      if (job.getNoOfBlackListedTrackers() > 0) {
        for (Entry<String, List<String>> e : job.getBlackListedTrackers().entrySet()) {
          String tt = e.getKey();
          String jobName = job.getJobID().toString();
          String[] exceptions = e.getValue().toArray(new String[0]);

          faultyTrackers.incrementFaults(tt, new JobFault(tt, jobName, exceptions));
        }
      }
    }

    String jobUser = job.getProfile().getUser();
    //add to the user to jobs mapping
    synchronized (userToJobsMap) {
      ArrayList<JobInProgress> userJobs = userToJobsMap.get(jobUser);
      if (userJobs == null) {
        userJobs =  new ArrayList<JobInProgress>();
        userToJobsMap.put(jobUser, userJobs);
      }
      userJobs.add(job);
    }
  }
46198
@SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
  private void waitForJobCompletion(final CommandSessionImpl session) throws InterruptedException {
    while (true) {
      Job job = session.foregroundJob();
      if (job == null) break;

      log.debug("Waiting for job completion: {}", job);
      synchronized (job) {
        if (job.status() == Job.Status.Foreground) {
          job.wait();
        }
      }
    }
  }
281277
protected void checkNext ()
    {
        _executingNow = !_queue.isEmpty();
        if (_executingNow) {
            // start up a thread to execute the task in question
            ExecutorTask task = _queue.remove(0);
            final ExecutorThread thread = new ExecutorThread(task);
            thread.start();

            // start up a timer that will abort this thread after the specified timeout
            new Interval(Interval.RUN_DIRECT) {
                @Override public void expired () {
                    // this will NOOP if the task has already completed
                    thread.abort();
                }
            }.schedule(task.getTimeout());
        }
    }
784688
protected void jobFinished(Throwable error)
    {
        this.lock.lock();

        try {
            if (this.status instanceof AbstractJobStatus) {
                // Store error
                ((AbstractJobStatus) this.status).setError(error);
            }

            // Give a chance to any listener to do custom action associated to the job
            this.observationManager.notify(new JobFinishingEvent(getRequest().getId(), getType(), this.request), this,
                error);

            if (getRequest().isVerbose()) {
                if (getStatus().getRequest().getId() != null) {
                    this.logger.info(LOG_END_ID, "Finished job of type [{}] with identifier [{}]", getType(),
                        getStatus().getRequest().getId());
                } else {
                    this.logger.info(LOG_END, "Finished job of type [{}]", getType());
                }
            }

            if (this.status instanceof AbstractJobStatus) {
                // Indicate when the job ended
                ((AbstractJobStatus) this.status).setEndDate(new Date());

                // Stop updating job status (progress, log, etc.)
                ((AbstractJobStatus) this.status).stopListening();

                // Update job state
                ((AbstractJobStatus) this.status).setState(JobStatus.State.FINISHED);
            }

            // Release threads waiting for job being done
            this.finishedCondition.signalAll();

            // Remove the job from the current jobs context
            this.jobContext.popCurrentJob();

            // Store the job status
            try {
                if (this.request.getId() != null) {
                    this.store.storeAsync(this.status);
                }
            } catch (Throwable t) {
                this.logger.warn(LOG_STATUS_STORE_FAILED, "Failed to store job status [{}]", this.status, t);
            }
        } finally {
            this.lock.unlock();

            // Notify listener that job is fully finished
            this.observationManager.notify(new JobFinishedEvent(getRequest().getId(), getType(), this.request), this,
                error);
        }
    }
886660
def terminate_all(self):
        
        logger.info('Job {0} terminating all currently running tasks'.format(self.name))
        for task in self.tasks.itervalues():
            if task.started_at and not task.completed_at:
                task.terminate()
621024
public synchronized boolean isJobComplete() {
     return (runState == JobStatus.SUCCEEDED || runState == JobStatus.FAILED 
             || runState == JobStatus.KILLED);
   }
991316
def killJobs(self, jobsToKill):
        
        if len(jobsToKill) > 0:
            self.batchSystem.killBatchJobs(jobsToKill)
            for jobBatchSystemID in jobsToKill:
                self.processFinishedJob(jobBatchSystemID, 1)
1224875
def check_workers(self):
        ''' '''
        if time.time() - self._worker_alive_time > 5:
            self._worker_alive_time = time.time()
            # join processes if they are now gone, it should not do anything bad
            # if the process is still running
            [worker.join() for worker in self._workers if not worker.is_alive()]
            self._workers = [
                worker for worker in self._workers if worker.is_alive()
            ]
            if len(self._workers) < self._num_workers:
                raise ProcessKilled('One of the workers has been killed.')
1320444
public void execute(Runnable job)
   {
      try
      {       
         executor.execute(job);
      }
      catch (RejectedExecutionException e)
      {
         log.warnf(e, "Job rejected: %s", job);
      }
   }
1132200
def finished(self):
    

    return self.__state in (Job.ERROR, Job.SUCCESS, Job.CANCELLED)
987310
def job_terminate(object_id, input_params={}, always_retry=True, **kwargs):
    
    return DXHTTPRequest('/%s/terminate' % object_id, input_params, always_retry=always_retry, **kwargs)
180699
def perform(job)
      begin
        if fork_per_job?
          reconnect
          run_hook :after_fork, job
        end
        job.perform
      rescue Object => e
        report_failed_job(job,e)
      else
        log_with_severity :info, "done: #{job.inspect}"
      ensure
        yield job if block_given?
      end
    end
876944
public function doneWorking()
    {
        $this->currentJob = null;
        $this->resque->getStatistic('processed')->incr();
        $this->getStatistic('processed')->incr();
        $this->resque->getClient()->del($this->getJobKey());
    }