Record worker completion status to the experiment server.

        This is done using a GET request to the /worker_complete
        or /worker_failed endpoints.
328684
def complete_experiment(self, status):
        
        self.log("Bot player completing experiment. Status: {}".format(status))
        while True:
            url = "{host}/{status}?participant_id={participant_id}".format(
                host=self.host, participant_id=self.participant_id, status=status
            )
            try:
                result = requests.get(url)
                result.raise_for_status()
            except RequestException:
                self.stochastic_sleep()
                continue
            return result
570237
def worker_finished(pid)
      worker_status(pid, 'stop')
      all_listeners.each do |other|
        other.worker_finished(pid)
      end
    end
1030451
public void noteRequestCompletion(ServerStats stats, Object response, Throwable e, long responseTime, RetryHandler errorHandler) {
    	if (stats == null) {
    		return;
    	}
        try {
            recordStats(stats, responseTime);
            RetryHandler callErrorHandler = errorHandler == null ? getRetryHandler() : errorHandler;
            if (callErrorHandler != null && response != null) {
                stats.clearSuccessiveConnectionFailureCount();
            } else if (callErrorHandler != null && e != null) {
                if (callErrorHandler.isCircuitTrippingException(e)) {
                    stats.incrementSuccessiveConnectionFailureCount();                    
                    stats.addToFailureCount();
                } else {
                    stats.clearSuccessiveConnectionFailureCount();
                }
            }
        } catch (Exception ex) {
            logger.error("Error noting stats for client {}", clientName, ex);
        }            
    }
1158546
public int completed(InetAddress request)
    {
        String dc = DatabaseDescriptor.getEndpointSnitch().getDatacenter(request);
        Queue<InetAddress> requests = requestsByDatacenter.get(dc);
        assert requests != null;
        assert request.equals(requests.peek());
        requests.poll();
        if (!requests.isEmpty())
            processor.process(requests.peek());
        return --remaining;
    }
54965
def end(self):
    
    if self._end_memoized_result is not None:
      return self._end_memoized_result
    if self._background_worker_pool:
      if self._aborted:
        self.log(Report.INFO, "Aborting background workers.")
        self._background_worker_pool.abort()
      else:
        self.log(Report.INFO, "Waiting for background workers to finish.")
        self._background_worker_pool.shutdown()
      self.end_workunit(self._background_root_workunit)

    self.shutdown_worker_pool()

    # Run a dummy work unit to write out one last timestamp.
    with self.new_workunit("complete"):
      pass

    self.end_workunit(self._main_root_workunit)

    outcome = self._main_root_workunit.outcome()
    if self._background_root_workunit:
      outcome = min(outcome, self._background_root_workunit.outcome())
    outcome_str = WorkUnit.outcome_string(outcome)
    log_level = RunTracker._log_levels[outcome]
    self.log(log_level, outcome_str)

    if self.run_info.get_info('outcome') is None:
      # If the goal is clean-all then the run info dir no longer exists, so ignore that error.
      self.run_info.add_info('outcome', outcome_str, ignore_errors=True)

    if self._target_to_data:
      self.run_info.add_info('target_data', self._target_to_data)

    self.report.close()
    self.store_stats()

    run_failed = outcome in [WorkUnit.FAILURE, WorkUnit.ABORTED]
    result = PANTS_FAILED_EXIT_CODE if run_failed else PANTS_SUCCEEDED_EXIT_CODE
    self._end_memoized_result = result
    return self._end_memoized_result
867491
def record_participation(self, client, dt=None):
        """"""
        if dt is None:
            date = datetime.now()
        else:
            date = dt

        experiment_key = self.experiment.name

        pipe = self.redis.pipeline()

        pipe.sadd(_key("p:{0}:years".format(experiment_key)), date.strftime('%Y'))
        pipe.sadd(_key("p:{0}:months".format(experiment_key)), date.strftime('%Y-%m'))
        pipe.sadd(_key("p:{0}:days".format(experiment_key)), date.strftime('%Y-%m-%d'))

        pipe.execute()

        keys = [
            _key("p:{0}:_all:all".format(experiment_key)),
            _key("p:{0}:_all:{1}".format(experiment_key, date.strftime('%Y'))),
            _key("p:{0}:_all:{1}".format(experiment_key, date.strftime('%Y-%m'))),
            _key("p:{0}:_all:{1}".format(experiment_key, date.strftime('%Y-%m-%d'))),
            _key("p:{0}:{1}:all".format(experiment_key, self.name)),
            _key("p:{0}:{1}:{2}".format(experiment_key, self.name, date.strftime('%Y'))),
            _key("p:{0}:{1}:{2}".format(experiment_key, self.name, date.strftime('%Y-%m'))),
            _key("p:{0}:{1}:{2}".format(experiment_key, self.name, date.strftime('%Y-%m-%d'))),
        ]
        msetbit(keys=keys, args=([self.experiment.sequential_id(client), 1] * len(keys)))
865652
def complete(self, nextq=None, delay=None, depends=None):
        ''''''
        if nextq:
            logger.info('Advancing %s to %s from %s',
                self.jid, nextq, self.queue_name)
            return self.client('complete', self.jid, self.client.worker_name,
                self.queue_name, json.dumps(self.data), 'next', nextq,
                'delay', delay or 0, 'depends',
                json.dumps(depends or [])) or False
        else:
            logger.info('Completing %s', self.jid)
            return self.client('complete', self.jid, self.client.worker_name,
                self.queue_name, json.dumps(self.data)) or False
635839
def _on_completion(self):
        

        logger.debug('Job {0} running _on_completion check'.format(self.name))
        if self.state.status != 'running' or (not self._is_complete()):
            return

        for job, results in self.run_log['tasks'].iteritems():
            if results.get('success', False) == False:
                self._set_status('failed')
                try:
                    self.backend.acquire_lock()
                    if self.event_handler:
                        self.event_handler.emit('job_failed',
                                                self._serialize(include_run_logs=True))
                except:
                    logger.exception("Error in handling events.")
                finally:
                    self.backend.release_lock()
                break

        if self.state.status != 'failed':
            self._set_status('waiting')
            self.run_log = {}
            try:
                self.backend.acquire_lock()
                if self.event_handler:
                    self.event_handler.emit('job_complete',
                                            self._serialize(include_run_logs=True))
            except:
                logger.exception("Error in handling events.")
            finally:
                self.backend.release_lock()

        self.destroy_snapshot()
820148
def _request_finished(self, reply):
        """"""
        url = to_text_string(reply.url().toEncoded(), encoding='utf-8')

        if url in self._paths:
            path = self._paths[url]
        if url in self._workers:
            worker = self._workers[url]

        if url in self._head_requests:
            error = reply.error()
#            print(url, error)
            if error:
                logger.error(str(('Head Reply Error:', error)))
                worker.sig_download_finished.emit(url, path)
                worker.sig_finished.emit(worker, path, error)
                return

            self._head_requests.pop(url)
            start_download = not bool(error)
            header_pairs = reply.rawHeaderPairs()
            headers = {}

            for hp in header_pairs:
                headers[to_text_string(hp[0]).lower()] = to_text_string(hp[1])

            total_size = int(headers.get('content-length', 0))

            # Check if file exists
            if os.path.isfile(path):
                file_size = os.path.getsize(path)

                # Check if existing file matches size of requested file
                start_download = file_size != total_size

            if start_download:
                # File sizes dont match, hence download file
                qurl = QUrl(url)
                request = QNetworkRequest(qurl)
                self._get_requests[url] = request
                reply = self._manager.get(request)

                error = reply.error()
                if error:
                    logger.error(str(('Reply Error:', error)))

                reply.downloadProgress.connect(
                    lambda r, t, w=worker: self._progress(r, t, w))
            else:
                # File sizes match, dont download file or error?
                worker.finished = True
                worker.sig_download_finished.emit(url, path)
                worker.sig_finished.emit(worker, path, None)
        elif url in self._get_requests:
            data = reply.readAll()
            self._save(url, path, data)
675160
def fetch_synchronize_status():
    
    r = Response()
    project = cd.project.get_internal_project()

    if not project:
        r.fail(
            code='NO_PROJECT',
            message='No open project on which to retrieve status'
        )
    else:
        with open(project.source_path, 'r') as f:
            definition = json.load(f)

        result = status.of_project(project)
        r.update(
            sync_time=sync_status.get('time', 0),
            source_directory=project.source_directory,
            remote_source_directory=project.remote_source_directory,
            status=result,
            definition=definition
        )

    return r.flask_serialize()
981273
def status(self):
        
        task_id = self.request.id
        try:
            # Most calls are for existing objects, don't waste time
            # preparing creation arguments unless necessary
            return UserTaskStatus.objects.get(task_id=task_id)
        except UserTaskStatus.DoesNotExist:
            # Probably an eager task that skipped the before_task_publish
            # signal (or an atomic view where the new record hasn't been
            # committed yet).  Create a record for it.
            arguments_dict = self.arguments_as_dict(*self.request.args, **self.request.kwargs)
            name = self.generate_name(arguments_dict)
            task_class = '.'.join([self.__class__.__module__, self.__class__.__name__])
            total_steps = self.calculate_total_steps(arguments_dict)
            user_id = arguments_dict['user_id']
            # Use get_or_create() again just in case another process created it in the meantime
            return UserTaskStatus.objects.get_or_create(
                task_id=task_id, defaults={'user_id': user_id, 'name': name, 'task_class': task_class,
                                           'total_steps': total_steps})[0]
619841
private <Q, R> void recordFinalStatus(JobIdentifier<Q, R> jobId, JobStatus<Q, R> jobStatus) {
        try {
            _jobStatusDAO.updateJobStatus(jobId, jobStatus);
        } catch (Exception e) {
            _log.error("Failed to record final status for job: [id={}, status={}]", jobId, jobStatus.getStatus(), e);
        }
    }
1328573
private void jobComplete() {
    final JobTrackerInstrumentation metrics = jobtracker.getInstrumentation();
    //
    // All tasks are complete, then the job is done!
    //
    if (this.status.getRunState() == JobStatus.RUNNING ||
        this.status.getRunState() == JobStatus.PREP) {
      changeStateTo(JobStatus.SUCCEEDED);
      this.status.setCleanupProgress(1.0f);
      if (maps.length == 0) {
        this.status.setMapProgress(1.0f);
      }
      if (reduces.length == 0) {
        this.status.setReduceProgress(1.0f);
      }
      this.finishTime = JobTracker.getClock().getTime();
      LOG.info("Job " + this.status.getJobID() +
               " has completed successfully.");

      // Log the job summary (this should be done prior to logging to
      // job-history to ensure job-counters are in-sync
      JobSummary.logJobSummary(this, jobtracker.getClusterStatus(false));

      Counters counters = getCounters();
      // Log job-history
      JobHistory.JobInfo.logFinished(this.status.getJobID(), finishTime,
                                     this.finishedMapTasks,
                                     this.finishedReduceTasks, failedMapTasks,
                                     failedReduceTasks, killedMapTasks,
                                     killedReduceTasks, getMapCounters(),
                                     getReduceCounters(), counters);
      // Note that finalize will close the job history handles which garbage collect
      // might try to finalize
      garbageCollect();

      metrics.completeJob(this.conf, this.status.getJobID());
    }
  }
518503
void recordOperationPrepareTimeout(final BlockingQueueOperationListener.FailedOperation<ServerOperation> failedOperation) {
        recordPreparedTask(new ServerTaskExecutor.ServerPreparedResponse(failedOperation));
        // Swap out the submitted task so we don't wait for the final result. Use a future the returns
        // prepared response
        ServerIdentity identity = failedOperation.getOperation().getIdentity();
        AsyncFuture<OperationResponse> finalResult = failedOperation.getFinalResult();
        synchronized (submittedTasks) {
            submittedTasks.put(identity, new ServerTaskExecutor.ExecutedServerRequest(identity, finalResult));
        }
    }
33258
private void taskComplete(
      HttpRemoteTaskRunnerWorkItem taskRunnerWorkItem,
      WorkerHolder workerHolder,
      TaskStatus taskStatus
  )
  {
    Preconditions.checkState(!Thread.holdsLock(statusLock), "Current thread must not hold statusLock.");
    Preconditions.checkNotNull(taskRunnerWorkItem, "taskRunnerWorkItem");
    Preconditions.checkNotNull(taskStatus, "taskStatus");
    if (workerHolder != null) {
      log.info(
          "Worker[%s] completed task[%s] with status[%s]",
          workerHolder.getWorker().getHost(),
          taskStatus.getId(),
          taskStatus.getStatusCode()
      );
      // Worker is done with this task
      workerHolder.setLastCompletedTaskTime(DateTimes.nowUtc());
    }

    // Notify interested parties
    taskRunnerWorkItem.setResult(taskStatus);
    TaskRunnerUtils.notifyStatusChanged(listeners, taskStatus.getId(), taskStatus);

    // Update success/failure counters, Blacklist node if there are too many failures.
    if (workerHolder != null) {
      blacklistWorkerIfNeeded(taskStatus, workerHolder);
    }

    synchronized (statusLock) {
      statusLock.notifyAll();
    }
  }
164747
def complete(self):
        
        return self._post_request(
            data='',
            endpoint=self.ENDPOINT + '/' + str(self.id) + '/complete'
        )
140540
public boolean waitForCompletion() {
        try {
            return completed.get();
        } catch (InterruptedException | ExecutionException e) {
            log.warn("Failed to wait for server completion", e);
        }

        return false;
    }