Update the preemption JobInfo fields for all jobs, i.e. the times since
each job last was at its guaranteed share and at > 1/2 of its fair share
for each type of task.
426054
private void updatePreemptionVariables() {
    long now = clock.getTime();
    for (Map.Entry<JobInProgress, JobInfo> entry: infos.entrySet()) {
      JobInProgress job = entry.getKey();
      JobInfo info = entry.getValue();
      if (job.getStatus().getRunState() != JobStatus.RUNNING) {
        // Job is still in PREP state and tasks aren't initialized. Count it as
        // both at min and fair share since we shouldn't start any timeouts now.
        info.lastTimeAtMapMinShare = now;
        info.lastTimeAtReduceMinShare = now;
        info.lastTimeAtMapHalfFairShare = now;
        info.lastTimeAtReduceHalfFairShare = now;
      } else {
        if (!isStarvedForMinShare(info, TaskType.MAP))
          info.lastTimeAtMapMinShare = now;
        if (!isStarvedForMinShare(info, TaskType.REDUCE))
          info.lastTimeAtReduceMinShare = now;
        if (!isStarvedForFairShare(info, TaskType.MAP))
          info.lastTimeAtMapHalfFairShare = now;
        if (!isStarvedForFairShare(info, TaskType.REDUCE))
          info.lastTimeAtReduceHalfFairShare = now;
      }
    }
  }
866316
@Override
  public synchronized void put(JobSpec jobSpec) {
    Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName()));
    Preconditions.checkNotNull(jobSpec);
    try {
      long startTime = System.currentTimeMillis();
      Path jobSpecPath = getPathForURI(this.jobConfDirPath, jobSpec.getUri());
      boolean isUpdate = fs.exists(jobSpecPath);

      materializedJobSpec(jobSpecPath, jobSpec, this.fs);
      this.mutableMetrics.updatePutJobTime(startTime);
      if (isUpdate) {
        this.listeners.onUpdateJob(jobSpec);
      } else {
        this.listeners.onAddJob(jobSpec);
      }
    } catch (IOException e) {
      throw new RuntimeException("When persisting a new JobSpec, unexpected issues happen:" + e.getMessage());
    } catch (JobSpecNotFoundException e) {
      throw new RuntimeException("When replacing a existed JobSpec, unexpected issue happen:" + e.getMessage());
    }
  }
661505
def update_cached_fields_pre_save(self, update_fields: list):
        
        if self.id and update_fields is None:
            self.update_cached_fields(commit=False, exceptions=False)
283044
public JobScheduleListHeaders withLastModified(DateTime lastModified) {
        if (lastModified == null) {
            this.lastModified = null;
        } else {
            this.lastModified = new DateTimeRfc1123(lastModified);
        }
        return this;
    }
1135357
public void setJobInProgress(Boolean workInProgress) {
        lastDateInfo = new SimpleDateFormat(FILENAME_DATE_PATTERN).format(new Date());
        this.jobInProgress = workInProgress;
    }
848770
private void updateFairShares(double totalSlots, final TaskType type) {
    // Find the proper ratio of (# of slots share / weight) by bineary search
    BinarySearcher searcher = new BinarySearcher() {
      @Override
      double targetFunction(double x) {
        return slotsUsedWithWeightToSlotRatio(x, type);
      }
    };
    double ratio = searcher.getSolution(totalSlots, lastWeightToFairShareRatio);
    lastWeightToFairShareRatio = ratio;

    // Set the fair shares based on the value of R we've converged to
    for (JobInfo info : infos.values()) {
      if (type == TaskType.MAP) {
        info.mapFairShare = computeShare(info, ratio, type);
      } else {
        info.reduceFairShare = computeShare(info, ratio, type);
      }
    }
  }
755563
def record_occupation_updates(self, updates, source, version):
        
        now = int(time.time())
        # Put it on the recordQueue and notify the worker thread.
        with self.recordCond:
            self.recordQueue.append((now, updates, source))
            self.recordCond.notify()
1199505
private void updateJobStats(Phase phase, ProcessingRates oldRates, 
      ProcessingRates newRates) {
    DataStatistics stats = job.getRunningTaskStatistics(phase);
    stats.updateStatistics(oldRates.getRate(phase), newRates.getRate(phase));
  }
984512
def update_job(self, job_record, uow, new_state):
        
        original_job_state = job_record.state
        job_record.state = new_state
        job_record.related_unit_of_work = uow.db_id
        self.job_dao.update(job_record)

        msg = 'Updated Job {0} for {1}@{2}: state transfer {3} -> {4};' \
              .format(job_record.db_id, job_record.process_name, job_record.timeperiod, original_job_state, new_state)
        self._log_message(INFO, job_record.process_name, job_record.timeperiod, msg)
803759
public JobScheduleExistsHeaders withLastModified(DateTime lastModified) {
        if (lastModified == null) {
            this.lastModified = null;
        } else {
            this.lastModified = new DateTimeRfc1123(lastModified);
        }
        return this;
    }
458359
def adjust_jobs_priority(self, high_value_jobs, priority=1):
        
        # Only job priorities that don't have an expiration date (2 weeks for new jobs or year 2100
        # for jobs update via load_preseed) are updated
        for jp in JobPriority.objects.filter(expiration_date__isnull=True):
            if jp.unique_identifier() not in high_value_jobs:
                if jp.priority != SETA_LOW_VALUE_PRIORITY:
                    logger.warning('Decreasing priority of %s', jp.unique_identifier())
                    jp.priority = SETA_LOW_VALUE_PRIORITY
                    jp.save(update_fields=['priority'])
            elif jp.priority != priority:
                logger.warning('Increasing priority of %s', jp.unique_identifier())
                jp.priority = priority
                jp.save(update_fields=['priority'])
1149714
def _updateJobWhenDone(self):
        

        def asyncUpdate():
            try:
                # Wait till all file writes have completed
                for i in range(len(self.workers)):
                    self.queue.put(None)

                for thread in self.workers:
                    thread.join()

                # Wait till input block-fn returns - in the event of an exception
                # this will eventually terminate
                self.inputBlockFn()

                # Check the terminate event, if set we can not guarantee
                # that the workers ended correctly, therefore we exit without
                # completing the update
                if self._terminateEvent.isSet():
                    raise RuntimeError("The termination flag is set, exiting before update")

                # Indicate any files that should be deleted once the update of
                # the job wrapper is completed.
                self.jobGraph.filesToDelete = list(self.filesToDelete)

                # Complete the job
                self.jobStore.update(self.jobGraph)

                # Delete any remnant jobs
                list(map(self.jobStore.delete, self.jobsToDelete))

                # Delete any remnant files
                list(map(self.jobStore.deleteFile, self.filesToDelete))

                # Remove the files to delete list, having successfully removed the files
                if len(self.filesToDelete) > 0:
                    self.jobGraph.filesToDelete = []
                    # Update, removing emptying files to delete
                    self.jobStore.update(self.jobGraph)
            except:
                self._terminateEvent.set()
                raise
            finally:
                # Indicate that _blockFn can return
                # This code will always run
                self.updateSemaphore.release()

        # The update semaphore is held while the job is written to the job store
        try:
            self.updateSemaphore.acquire()
            t = Thread(target=asyncUpdate)
            t.start()
        except:
            # This is to ensure that the semaphore is released in a crash to stop a deadlock
            # scenario
            self.updateSemaphore.release()
            raise
188086
public JobScheduleAddHeaders withLastModified(DateTime lastModified) {
        if (lastModified == null) {
            this.lastModified = null;
        } else {
            this.lastModified = new DateTimeRfc1123(lastModified);
        }
        return this;
    }
625897
@Scheduled(fixedRate = KEEP_LAST_JOBS_CLEANUP_INTERVAL)
    public void doCleanUp() {
        final List<JobInfo> jobs = jobRepository.findAllJobInfoWithoutMessages();

        findJobsToDelete(jobs)
                .forEach(jobInfo -> jobRepository.removeIfStopped(jobInfo.getJobId()));
    }
152664
void markCompletedJob(JobInProgress job) {
    for (TaskInProgress tip : job.getTasks(TaskType.JOB_SETUP)) {
      for (TaskStatus taskStatus : tip.getTaskStatuses()) {
        if (taskStatus.getRunState() != TaskStatus.State.RUNNING &&
            taskStatus.getRunState() != TaskStatus.State.COMMIT_PENDING &&
            taskStatus.getRunState() != TaskStatus.State.UNASSIGNED) {
          markCompletedTaskAttempt(taskStatus.getTaskTracker(),
                                   taskStatus.getTaskID());
        }
      }
    }
    for (TaskInProgress tip : job.getTasks(TaskType.MAP)) {
      for (TaskStatus taskStatus : tip.getTaskStatuses()) {
        if (taskStatus.getRunState() != TaskStatus.State.RUNNING &&
            taskStatus.getRunState() != TaskStatus.State.COMMIT_PENDING &&
            taskStatus.getRunState() != TaskStatus.State.FAILED_UNCLEAN &&
            taskStatus.getRunState() != TaskStatus.State.KILLED_UNCLEAN &&
            taskStatus.getRunState() != TaskStatus.State.UNASSIGNED) {
          markCompletedTaskAttempt(taskStatus.getTaskTracker(),
                                   taskStatus.getTaskID());
        }
      }
    }
    for (TaskInProgress tip : job.getTasks(TaskType.REDUCE)) {
      for (TaskStatus taskStatus : tip.getTaskStatuses()) {
        if (taskStatus.getRunState() != TaskStatus.State.RUNNING &&
            taskStatus.getRunState() != TaskStatus.State.COMMIT_PENDING &&
            taskStatus.getRunState() != TaskStatus.State.FAILED_UNCLEAN &&
            taskStatus.getRunState() != TaskStatus.State.KILLED_UNCLEAN &&
            taskStatus.getRunState() != TaskStatus.State.UNASSIGNED) {
          markCompletedTaskAttempt(taskStatus.getTaskTracker(),
                                   taskStatus.getTaskID());
        }
      }
    }
  }
759197
def clear_expiration_field_for_expired_jobs(self):
        ''''''
        # Only select rows where there is an expiration date set
        for job in JobPriority.objects.filter(expiration_date__isnull=False):
            if job.has_expired():
                job.expiration_date = None
                job.save()
1155231
@Override
    public WSJobInstance updateJobInstanceAndExecutionWithInstanceStateAndBatchStatus(long instanceId, long executionId,
                                                                                      final InstanceState state, final BatchStatus batchStatus) {

        JobInstanceEntity retMe = null;

        //Update the JobInstance
        retMe = (JobInstanceEntity) persistenceManagerService.updateJobInstanceWithInstanceStateAndBatchStatus(instanceId, state, batchStatus, new Date());

        //Update the Job Execution Instance
        persistenceManagerService.updateJobExecutionAndInstanceOnStatusChange(executionId, batchStatus, new Date());

        return retMe;
    }