Checks whether any completed tasks returned before scanning the entire range.  If so then the unscanned
ranges are resplit and new tasks are created from them.
436618
private ScanStatus resplitPartiallyCompleteTasks(ScanStatus status) {
        boolean anyUpdated = false;
        int nextTaskId = -1;

        for (ScanRangeStatus complete : status.getCompleteScanRanges()) {
            if (complete.getResplitRange().isPresent()) {
                // This task only partially completed; there are still more data to scan.
                if (nextTaskId == -1) {
                    nextTaskId = getNextTaskId(status);
                }

                ScanRange resplitRange = complete.getResplitRange().get();

                // Resplit the un-scanned portion into new ranges
                List<ScanRange> subRanges = resplit(complete.getPlacement(), resplitRange, status.getOptions().getRangeScanSplitSize());
                // Create new tasks for each subrange that are immediately available for being queued.
                List<ScanRangeStatus> resplitStatuses = Lists.newArrayListWithCapacity(subRanges.size());
                for (ScanRange subRange : subRanges) {
                    resplitStatuses.add(
                            new ScanRangeStatus(nextTaskId++, complete.getPlacement(), subRange,
                                    complete.getBatchId(), complete.getBlockedByBatchId(), complete.getConcurrencyId()));
                }

                _scanStatusDAO.resplitScanRangeTask(status.getScanId(), complete.getTaskId(), resplitStatuses);

                anyUpdated = true;
            }
        }

        if (!anyUpdated) {
            return status;
        }

        // Slightly inefficient to reload but less risky than trying to keep the DAO and in-memory object in sync
        return _scanStatusDAO.getScanStatus(status.getScanId());
    }
82729
def task_done(self, **kw):
        
        def validate(task):
            if not Status.is_pending(task['status']):
                raise ValueError("Task is not pending.")

        return self._task_change_status(Status.COMPLETED, validate, **kw)
432590
public synchronized boolean completedTask(TaskInProgress tip,
                                            TaskStatus status)
  {
    TaskAttemptID taskid = status.getTaskID();
    int oldNumAttempts = tip.getActiveTasks().size();
    final JobTrackerInstrumentation metrics = jobtracker.getInstrumentation();

    // Metering
    meterTaskAttempt(tip, status);

    // Sanity check: is the TIP already complete?
    // It _is_ safe to not decrement running{Map|Reduce}Tasks and
    // finished{Map|Reduce}Tasks variables here because one and only
    // one task-attempt of a TIP gets to completedTask. This is because
    // the TaskCommitThread in the JobTracker marks other, completed,
    // speculative tasks as _complete_.
    if (tip.isComplete()) {
      // Mark this task as KILLED
      tip.alreadyCompletedTask(taskid);

      // Let the JobTracker cleanup this taskid if the job isn't running
      if (this.status.getRunState() != JobStatus.RUNNING) {
        jobtracker.markCompletedTaskAttempt(status.getTaskTracker(), taskid);
      }
      return false;
    }


    LOG.info("Task '" + taskid + "' has completed " + tip.getTIPId() +
             " successfully.");

    // Mark the TIP as complete
    tip.completed(taskid);
    resourceEstimator.updateWithCompletedTask(status, tip);

    // Update jobhistory
    TaskTrackerStatus ttStatus =
      this.jobtracker.getTaskTrackerStatus(status.getTaskTracker());
    String trackerHostname = jobtracker.getNode(ttStatus.getHost()).toString();
    String taskType = getTaskType(tip);
    if (status.getIsMap()){
      JobHistory.MapAttempt.logStarted(status.getTaskID(), status.getStartTime(),
                                       status.getTaskTracker(),
                                       ttStatus.getHttpPort(),
                                       taskType);
      JobHistory.MapAttempt.logFinished(status.getTaskID(), status.getFinishTime(),
                                        trackerHostname, taskType,
                                        status.getStateString(),
                                        status.getCounters());
    }else{
      JobHistory.ReduceAttempt.logStarted( status.getTaskID(), status.getStartTime(),
                                          status.getTaskTracker(),
                                          ttStatus.getHttpPort(),
                                          taskType);
      JobHistory.ReduceAttempt.logFinished(status.getTaskID(), status.getShuffleFinishTime(),
                                           status.getSortFinishTime(), status.getFinishTime(),
                                           trackerHostname,
                                           taskType,
                                           status.getStateString(),
                                           status.getCounters());
    }
    JobHistory.Task.logFinished(tip.getTIPId(),
                                taskType,
                                tip.getExecFinishTime(),
                                status.getCounters());

    int newNumAttempts = tip.getActiveTasks().size();
    if (tip.isJobSetupTask()) {
      // setup task has finished. kill the extra setup tip
      killSetupTip(!tip.isMapTask());
      setupComplete();
    } else if (tip.isJobCleanupTask()) {
      // cleanup task has finished. Kill the extra cleanup tip
      if (tip.isMapTask()) {
        // kill the reduce tip
        cleanup[1].kill();
      } else {
        cleanup[0].kill();
      }

      //
      // The Job is done
      // if the job is failed, then mark the job failed.
      if (jobFailed) {
        terminateJob(JobStatus.FAILED);
      }
      // if the job is killed, then mark the job killed.
      if (jobKilled) {
        terminateJob(JobStatus.KILLED);
      }
      else {
        jobComplete();
      }
      // The job has been killed/failed/successful
      // JobTracker should cleanup this task
      jobtracker.markCompletedTaskAttempt(status.getTaskTracker(), taskid);
    } else if (tip.isMapTask()) {
      // check if this was a sepculative task
      if (oldNumAttempts > 1) {
        speculativeMapTasks -= (oldNumAttempts - newNumAttempts);
        if (!garbageCollected) {
          totalSpeculativeMapTasks.addAndGet(newNumAttempts - oldNumAttempts);
        }
      }
      if (tip.isSpeculativeAttempt(taskid)) {
        metrics.speculativeSucceededMap(taskid,
            tip.isUsingProcessingRateForSpeculation());
      }
      int level = getLocalityLevel(tip, ttStatus);
      long inputBytes = tip.getCounters()
                .getGroup("org.apache.hadoop.mapred.Task$Counter")
                .getCounter("Map input bytes");
      switch (level) {
      case 0: jobCounters.incrCounter(Counter.LOCAL_MAP_INPUT_BYTES,
                inputBytes);
              metrics.addLocalMapInputBytes(inputBytes);
              break;
      case 1: jobCounters.incrCounter(Counter.RACK_MAP_INPUT_BYTES,
                inputBytes);
              metrics.addRackMapInputBytes(inputBytes);
              break;
      default:metrics.addMapInputBytes(inputBytes);
              break;
      }
      finishedMapTasks += 1;
      metrics.completeMap(taskid);
      if (!garbageCollected) {
        if (!tip.isJobSetupTask() && hasSpeculativeMaps) {
          updateTaskTrackerStats(tip,ttStatus,trackerMapStats,mapTaskStats);
        }
      }
      // remove the completed map from the resp running caches
      retireMap(tip);
      if ((finishedMapTasks + failedMapTIPs) == (numMapTasks)) {
        this.status.setMapProgress(1.0f);
      }
    } else {
      if (oldNumAttempts > 1) {
        speculativeReduceTasks -= (oldNumAttempts - newNumAttempts);
        if (!garbageCollected) {
          totalSpeculativeReduceTasks.addAndGet(newNumAttempts - oldNumAttempts);
        }
      }
      if (tip.isSpeculativeAttempt(taskid)) {
        metrics.speculativeSucceededReduce(taskid,
            tip.isUsingProcessingRateForSpeculation());
      }
      finishedReduceTasks += 1;
      metrics.completeReduce(taskid);
      if (!garbageCollected) {
        if (!tip.isJobSetupTask() && hasSpeculativeReduces) {
          updateTaskTrackerStats(tip,ttStatus,trackerReduceStats,reduceTaskStats);
        }
      }
      // remove the completed reduces from the running reducers set
      retireReduce(tip);
      if ((finishedReduceTasks + failedReduceTIPs) == (numReduceTasks)) {
        this.status.setReduceProgress(1.0f);
      }
    }

    // is job complete?
    if (!jobSetupCleanupNeeded && canLaunchJobCleanupTask()) {
      jobComplete();
    }

    return true;
  }
609402
private void checkForCompletion() {
        final String methodName = "checkForCompletion(): ";
        // mark the list complete if that was the last awaited key
        if (actualIndices.isEmpty()) {
            if (tc.isDebugEnabled())
                Tr.debug(tc, methodName + "setting state=COMPLETE");
            state = failedKeys == null
                            ? COMPLETE
                            : COMPLETE_WITH_FAILURES; // state change must be last action
        }
    }
1106193
protected synchronized void postTaskExecution() {
		if (stopped) {
			// Stopped, so don't count the task(s) as done.
			// (worker threads call this method even if the task was not really executed.)
			return;
		}
		tasksDoneCount++;
		int percentageComplete = tasksDoneCount * 100 / tasksTotalCount;

		// Compute the progress and notify the listeners
		this.notifyListenersSpiderProgress(percentageComplete, tasksDoneCount, tasksTotalCount - tasksDoneCount);

		// Check for ending conditions
		if (tasksDoneCount == tasksTotalCount && initialized) {
			this.complete();
		}
	}
358224
void recomputeProgress() {
    if (isComplete()) {
      this.progress = 1;
      // update the counters and the state
      TaskStatus completedStatus = taskStatuses.get(getSuccessfulTaskid());
      this.counters = completedStatus.getCounters();
      this.state = completedStatus.getStateString();
    } else if (failed) {
      this.progress = 0;
      // reset the counters and the state
      this.state = "";
      this.counters = new Counters();
    } else {
      double bestProgress = 0;
      String bestState = "";
      Counters bestCounters = new Counters();
      for (Iterator<TaskAttemptID> it = taskStatuses.keySet().iterator(); it.hasNext();) {
        TaskAttemptID taskid = it.next();
        TaskStatus status = taskStatuses.get(taskid);
        if (status.getRunState() == TaskStatus.State.SUCCEEDED) {
          bestProgress = 1;
          bestState = status.getStateString();
          bestCounters = status.getCounters();
          break;
        } else if (status.getRunState() == TaskStatus.State.COMMIT_PENDING) {
          //for COMMIT_PENDING, we take the last state that we recorded
          //when the task was RUNNING
          bestProgress = this.progress;
          bestState = this.state;
          bestCounters = this.counters;
        } else if (status.getRunState() == TaskStatus.State.RUNNING) {
          if (status.getProgress() >= bestProgress) {
            bestProgress = status.getProgress();
            bestState = status.getStateString();
            if (status.getIncludeCounters()) {
              bestCounters = status.getCounters();
            } else {
              bestCounters = this.counters;
            }
          }
        }
      }
      this.progress = bestProgress;
      this.state = bestState;
      this.counters = bestCounters;
    }
  }
912708
function filterFilesForUnchangedTasks(before, after, paths) {
  var pathsRes = [];
  var tasksRes = [];

  for (var i = 0; i < before.length; i++) {
    var unchangedTasks = filterUnchangedTasks(before[i], after[i]);
    if (unchangedTasks.length > 0) {
      tasksRes.push(unchangedTasks);
      pathsRes.push(paths[i]);
    }
  }

  return [tasksRes, pathsRes];
}
1339066
private Date extrapolateStartTimeFromScanRanges(List<ScanRangeStatus> pendingScanRanges,
                                                    List<ScanRangeStatus> activeScanRanges,
                                                    List<ScanRangeStatus> completeScanRanges) {
        Date startTime = null;

        for (ScanRangeStatus status : Iterables.concat(pendingScanRanges, activeScanRanges, completeScanRanges)) {
            Date queuedTime = status.getScanQueuedTime();
            if (queuedTime != null && (startTime == null || queuedTime.before(startTime))) {
                startTime = queuedTime;
            }
        }

        if (startTime == null) {
            // Either the scan contained no scan ranges or no scans were ever queued, neither of which is likely to
            // ever happen.  But, so we don't return null choose a reasonably old date.
            startTime = new Date(0);
        }

        return startTime;
    }
573846
public static void finalizePrecursorWindows(IScanCollection scans) {
    TreeMap<Integer, IntervalST<Double, TreeMap<Integer, IScan>>> mapMsLevel2rangeGroups = scans
        .getMapMsLevel2rangeGroups();

    List<Integer> msLevelsToRemove = new ArrayList<>();

    msLevelLoop:
    for (Map.Entry<Integer, IntervalST<Double, TreeMap<Integer, IScan>>> entry : mapMsLevel2rangeGroups
        .entrySet()) {

      int msLevel = entry.getKey();
      IntervalST<Double, TreeMap<Integer, IScan>> rangeMapMS2 = entry.getValue();

      if (rangeMapMS2.size() == 1) {
        msLevelsToRemove.add(msLevel);
      } else {
        // if we have a precursor range map, we want to see if it actually contains
        // ranges that have lots of MS2 spectra in them.
        // ALL ranges are required to have more than 1 MS2 spectra

        for (IntervalST.Node<Double, TreeMap<Integer, IScan>> node : rangeMapMS2) {
          if (node.getValue().size() <= 1) {
            // if we didn't have at least 1 scans in even one range, then discard the whole sub-tree
            msLevelsToRemove.add(msLevel);
            continue msLevelLoop;
          }
        }

        // we also check that all ranges contain approximately equal amount of scans
        // THIS WILL NOT WORK FOR MSX DATA, UNLESS "maxScanCountDiff" is set to some large value, which
        // should be at least [ms1_mz_range / (msx_isolation_width * msx_multiplex_number)]
        if (!isAllRangesHaveApproxSameScanCounts(rangeMapMS2)) {
          msLevelsToRemove.add(msLevel);
          continue msLevelLoop;
        }

        // otherwise we just keep it there, but the tree needs to be recreated, because the ranges are incorrect
      }
    }
      for (Integer msLevel : msLevelsToRemove) {
          mapMsLevel2rangeGroups.remove(msLevel);
      }
  }
1135858
private void ensureNext() {
        // Check if the current scan result has more keys (i.e. the index did not reach the end of the result list)
        if (resultIndex < scanResult.getResult().size()) {
            return;
        }
        // Since the current scan result was fully iterated,
        // if there is another cursor scan it and ensure next key (recursively)
        if (!FIRST_CURSOR.equals(scanResult.getStringCursor())) {
            scanResult = scan(scanResult.getStringCursor(), scanParams);
            resultIndex = 0;
            ensureNext();
        }
    }
255282
def check_complete(self):
        
        logger.debug('Running check_complete for task {0}'.format(self.name))

        # Tasks not completed
        if self.remote_not_complete() or self.local_not_complete():
            self._start_check_timer()
            return

        return_code = self.completed_task()

        # Handle task errors
        if self.terminate_sent:
            self.stderr += '\nDAGOBAH SENT SIGTERM TO THIS PROCESS\n'
        if self.kill_sent:
            self.stderr += '\nDAGOBAH SENT SIGKILL TO THIS PROCESS\n'
        if self.remote_failure:
            return_code = -1
            self.stderr += '\nAn error occurred with the remote machine.\n'

        self.stdout_file = None
        self.stderr_file = None

        self._task_complete(success=True if return_code == 0 else False,
                            return_code=return_code,
                            stdout=self.stdout,
                            stderr=self.stderr,
                            start_time=self.started_at,
                            complete_time=datetime.utcnow())
1049910
def task_done(self):
        
        if not self._open_tasks:
            raise ValueError("task_done() called too many times")
        self._open_tasks -= 1
        if not self._open_tasks:
            self._jobs_done.set()
264542
private List<ClaimedTask> claimMigrationRangeTasks(int max) {
        try {
            Date claimTime = new Date();
            List<ScanRangeTask> migrationRangeTasks = _workflow.claimScanRangeTasks(max, QUEUE_CLAIM_TTL);
            if (migrationRangeTasks.isEmpty()) {
                return ImmutableList.of();
            }

            List<ClaimedTask> newlyClaimedTasks = Lists.newArrayListWithCapacity(migrationRangeTasks.size());

            for (ScanRangeTask task : migrationRangeTasks) {
                final ClaimedTask claimedTask = new ClaimedTask(task, claimTime);

                // Record that the task is claimed locally
                boolean alreadyClaimed = _claimedTasks.putIfAbsent(task.getId(), claimedTask) != null;
                if (alreadyClaimed) {
                    _log.warn("Workflow returned migration range task that is already claimed: {}", task);
                    // Do not acknowledge the task, let it expire naturally.  Eventually it should come up again
                    // after the previous claim has been released.
                } else {
                    _log.info("Claimed migration range task: {}", task);
                    newlyClaimedTasks.add(claimedTask);

                    // Schedule a follow-up to ensure the scanning service assigns it a thread
                    // in a reasonable amount of time.
                    _backgroundService.schedule(
                            new Runnable() {
                                @Override
                                public void run() {
                                    validateClaimedTaskHasStarted(claimedTask);
                                }
                            },
                            CLAIM_START_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
                }
            }

            return newlyClaimedTasks;
        } catch (Exception e) {
            _log.error("Failed to start next available migration range", e);
            return ImmutableList.of();
        }
    }
286086
public void markTaskCompletion() {
    if (this.countDownLatch.isPresent()) {
      this.countDownLatch.get().countDown();
    }

    this.taskState.setProp(ConfigurationKeys.TASK_RETRIES_KEY, this.retryCount.get());
  }
301523
@VisibleForTesting
  void checkSegmentsAndSubmitTasks()
  {
    synchronized (taskLock) {
      List<Interval> intervalsToRemove = new ArrayList<>();
      for (Map.Entry<Interval, HadoopIndexTask> entry : runningTasks.entrySet()) {
        Optional<TaskStatus> taskStatus = taskStorage.getStatus(entry.getValue().getId());
        if (!taskStatus.isPresent() || !taskStatus.get().isRunnable()) {
          intervalsToRemove.add(entry.getKey());
        }
      }
      for (Interval interval : intervalsToRemove) {
        runningTasks.remove(interval);
        runningVersion.remove(interval);
      }

      if (runningTasks.size() == maxTaskCount) {
        //if the number of running tasks reach the max task count, supervisor won't submit new tasks.
        return;
      }
      Pair<SortedMap<Interval, String>, Map<Interval, List<DataSegment>>> toBuildIntervalAndBaseSegments = checkSegments();
      SortedMap<Interval, String> sortedToBuildVersion = toBuildIntervalAndBaseSegments.lhs;
      Map<Interval, List<DataSegment>> baseSegments = toBuildIntervalAndBaseSegments.rhs;
      missInterval = sortedToBuildVersion.keySet();
      submitTasks(sortedToBuildVersion, baseSegments);
    }
  }
120402
def check_for_missed_cleanup():
    
    if get_setting('PRESERVE_ALL'):
        return
    from api.models.tasks import TaskAttempt
    if get_setting('PRESERVE_ON_FAILURE'):
        for task_attempt in TaskAttempt.objects.filter(
                status_is_running=False).filter(
                    status_is_cleaned_up=False).exclude(
                        status_is_failed=True):
            task_attempt.cleanup()
    else:
        for task_attempt in TaskAttempt.objects.filter(
                status_is_running=False).filter(status_is_cleaned_up=False):
            task_attempt.cleanup()
19300
def complete(self):
        
        if self.scan_limit is not None and self.scan_limit == 0:
            return True
        if self.item_limit is not None and self.item_limit == 0:
            return True
        return False