Handle cron healthcheck

Restart the background process if not already running
and data exists in the queue.
331411
public function handle_cron_healthcheck() {
		if ( $this->is_process_running() ) {
			// Background process already running.
			exit;
		}

		if ( $this->is_queue_empty() ) {
			// No data to process.
			$this->clear_scheduled_event();
			exit;
		}

		$this->handle();

		exit;
	}
347686
def run_get_data_background(macs, queue):
    

    def callback(data):
        data[1]['time'] = str(datetime.now())
        queue.put(data)

    RuuviTagSensor.get_datas(callback, macs)
1039169
public void waitOnBackgroundOp()
  {
    ExecutorService jobRunner = getJobRunner();
    if( jobRunner != null )
    {
      final Object wait = new Object();
      synchronized( wait ) //## wtf?
      {
        BackgroundOperation.instance().doBackgroundOp( () -> {
          synchronized( wait )
          {
            wait.notifyAll();
          }
        }, false );

        try
        {
          wait.wait();
        }
        catch( InterruptedException e )
        {
          //ignore
        }
      }
    }
  }
1272383
async def ensure_process(self):
        
        # We don't want multiple requests trying to start the process at the same time
        # FIXME: Make sure this times out properly?
        # Invariant here should be: when lock isn't being held, either 'proc' is in state &
        # running, or not.
        with (await self.state['proc_lock']):
            if 'proc' not in self.state:
                # FIXME: Prevent races here
                # FIXME: Handle graceful exits of spawned processes here
                cmd = self.get_cmd()
                server_env = os.environ.copy()

                # Set up extra environment variables for process
                server_env.update(self.get_env())

                timeout = self.get_timeout()

                proc = SupervisedProcess(self.name, *cmd, env=server_env, ready_func=self._http_ready_func, ready_timeout=timeout, log=self.log)
                self.state['proc'] = proc

                try:
                    await proc.start()

                    is_ready = await proc.ready()

                    if not is_ready:
                        await proc.kill()
                        raise web.HTTPError(500, 'could not start {} in time'.format(self.name))
                except:
                    # Make sure we remove proc from state in any error condition
                    del self.state['proc']
                    raise
700239
function restart() {
	logger.log('Restarting', clc.green(script));
	notifyWebSocket('restart');
	if (!kill(true)) {
		// The process wasn't running, start it now.
		start(true);
	} /*else {
		// The process will restart when its 'exit' event is emitted.
	}*/
}
390670
public function start()
    {
        $this->logger->info("crontab start");
        $crontab = $this->createCrontab();
        $loop = Factory::create();

        // add periodic timer
        $loop->addPeriodicTimer(60, function () use ($crontab, $loop) {
            $loop->addTimer(60 - time() % 60, function () use ($crontab) {
                $pid = pcntl_fork();
                if ($pid > 0) {
                    return;
                } elseif ($pid == 0) {
                    $crontab->start(time());
                    exit();
                } else {
                    $this->logger->error("could not fork");
                }
            });
        });

        // recover the sub processes
        $loop->addPeriodicTimer(60, function () {
            while (($pid = pcntl_waitpid(0, $status, WNOHANG)) > 0) {
                $message = "process exit. pid:" . $pid . ". exit code:" . $status;
                $this->logger->info($message);
            }
        });

        $loop->run();
    }
407609
protected function housekeeping_check()
	{
		if ((time() - $this->housekeeping_last_check) >= $this->housekeeping_check_interval)
		{
			// check to make sure no children are violating the max run time
			$this->kill_maxtime_violators();

			// look for zombie children just in case
			$this->signal_handler_sigchild(SIGCHLD);

			// update the last check time to now
			$this->housekeeping_last_check = time();
		}
	}
577731
def ensure_on():
    
    if get_status() == 'not-running':
        if config.dbserver.multi_user:
            sys.exit('Please start the DbServer: '
                     'see the documentation for details')
        # otherwise start the DbServer automatically; NB: I tried to use
        # multiprocessing.Process(target=run_server).start() and apparently
        # it works, but then run-demos.sh hangs after the end of the first
        # calculation, but only if the DbServer is started by oq engine (!?)
        subprocess.Popen([sys.executable, '-m', 'openquake.server.dbserver',
                          '-l', 'INFO'])

        # wait for the dbserver to start
        waiting_seconds = 30
        while get_status() == 'not-running':
            if waiting_seconds == 0:
                sys.exit('The DbServer cannot be started after 30 seconds. '
                         'Please check the configuration')
            time.sleep(1)
            waiting_seconds -= 1
1031077
def run(self):
        
        self._capture_signals()
        self._start_monitor()
        try:
            while True:
                if not self._run_worker():
                    self._wait_for_changes()
                time.sleep(self.reload_interval)
        except KeyboardInterrupt:
            pass
        finally:
            self._stop_monitor()
            self._restore_signals()
        sys.exit(1)
653834
def StartRun(self, wait_for_start_event, signal_event, wait_for_write_event):
    """"""
    # Signal that the cron thread has started. This way the cron scheduler
    # will know that the task is not sitting in a threadpool queue, but is
    # actually executing.
    wait_for_start_event.set()
    # Wait until the cron scheduler acknowledges the run. If it doesn't
    # acknowledge, just return (it means that the cron scheduler considers
    # this task as "not started" and has returned the lease so that another
    # worker can pick it up).
    if not signal_event.wait(TASK_STARTUP_WAIT):
      return

    try:
      logging.info("Processing cron job: %s", self.job.cron_job_id)
      self.run_state.started_at = rdfvalue.RDFDatetime.Now()
      self.run_state.status = "RUNNING"

      data_store.REL_DB.WriteCronJobRun(self.run_state)
      data_store.REL_DB.UpdateCronJob(
          self.job.cron_job_id,
          last_run_time=rdfvalue.RDFDatetime.Now(),
          current_run_id=self.run_state.run_id,
          forced_run_requested=False)
    finally:
      # Notify the cron scheduler that all the DB updates are done. At this
      # point the cron scheduler can safely return this job's lease.
      wait_for_write_event.set()

    try:
      self.Run()
      self.run_state.status = "FINISHED"
    except LifetimeExceededError:
      self.run_state.status = "LIFETIME_EXCEEDED"
      stats_collector_instance.Get().IncrementCounter(
          "cron_job_failure", fields=[self.job.cron_job_id])
    except Exception as e:  # pylint: disable=broad-except
      logging.exception("Cronjob %s failed with an error: %s",
                        self.job.cron_job_id, e)
      stats_collector_instance.Get().IncrementCounter(
          "cron_job_failure", fields=[self.job.cron_job_id])
      self.run_state.status = "ERROR"
      self.run_state.backtrace = str(e)

    finally:
      self.run_state.finished_at = rdfvalue.RDFDatetime.Now()
      elapsed = self.run_state.finished_at - self.run_state.started_at
      stats_collector_instance.Get().RecordEvent(
          "cron_job_latency", elapsed.seconds, fields=[self.job.cron_job_id])
      if self.job.lifetime:
        expiration_time = self.run_state.started_at + self.job.lifetime
        if self.run_state.finished_at > expiration_time:
          self.run_state.status = "LIFETIME_EXCEEDED"
          stats_collector_instance.Get().IncrementCounter(
              "cron_job_timeout", fields=[self.job.cron_job_id])
      data_store.REL_DB.WriteCronJobRun(self.run_state)

    current_job = data_store.REL_DB.ReadCronJob(self.job.cron_job_id)
    # If no other job was started while we were running, update last status
    # information.
    if current_job.current_run_id == self.run_state.run_id:
      data_store.REL_DB.UpdateCronJob(
          self.job.cron_job_id,
          current_run_id=None,
          last_run_status=self.run_state.status)
555738
function backgroundRefresh(){
    for(key in refreshKeys){
      if(refreshKeys.hasOwnProperty(key)){
        var data = refreshKeys[key];
        if(data.expiration - Date.now() < self.backgroundRefreshMinTtl){
          data.refresh(key, function (err, response){
            if(!err){
              self.set(key, response, data.lifeSpan, data.refresh, noop);
            }
          });
        }
      }
    }
  }
1258472
def _monitor(self):
        
        err_msg = ("invalid internal state:"
                   " _stop_nowait can not be set if _stop is not set")
        assert self._stop.isSet() or not self._stop_nowait.isSet(), err_msg

        q = self.queue
        has_task_done = hasattr(q, 'task_done')
        while not self._stop.isSet():
            try:
                record = self.dequeue(True)
                if record is self._sentinel_item:
                    break
                self.handle(record)
                if has_task_done:
                    q.task_done()
            except queue.Empty:
                pass

        # There might still be records in the queue,
        # handle then unless _stop_nowait is set.
        while not self._stop_nowait.isSet():
            try:
                record = self.dequeue(False)
                if record is self._sentinel_item:
                    break
                self.handle(record)
                if has_task_done:
                    q.task_done()
            except queue.Empty:
                break
638514
function monitor() {
    if(!child.pid) {
      // If the number of periodic starts exceeds the max, kill the process
      if (starts >= options.maxRetries) {
        if ((Date.now() - startTime) > maxMilliseconds) {
          console.error(
            `Too many restarts within the last ${maxMilliseconds / 1000} seconds. Please check the script.`
          )
          process.exit(1)
        }
      }

      setTimeout(() => {
        wait = wait * options.growPercentage
        attempts += 1;
        if (attempts > options.maxRestarts && options.maxRestarts >= 0){
          console.error(
            `${options.name} will not be restarted because the maximum number of total restarts has been exceeded.`
          )
          process.exit()
        } else {
          launch()
        }
      }, wait)
    } else {
      attempts = 0
      wait = options.restartDelay * 1000
    }
  }
347425
public function handleRefresh()
    {
        if ($this->hasPendingJob()) {
            return;
        }

        // Queue the job in the immediate queue
        $job = Injector::inst()->create(CheckForUpdatesJob::class);
        $jobDescriptorId = $this->getQueuedJobService()->queueJob($job, null, null, QueuedJob::IMMEDIATE);

        // Check the job descriptor on the queue
        $jobDescriptor = QueuedJobDescriptor::get()->filter('ID', $jobDescriptorId)->first();

        // If the job is not immediate, change it to immediate and reschedule it to occur immediately
        if ($jobDescriptor->JobType !== QueuedJob::IMMEDIATE) {
            $jobDescriptor->JobType = QueuedJob::IMMEDIATE;
            $jobDescriptor->StartAfter = null;
            $jobDescriptor->write();
        }
    }
547873
def restart(self, restart_data, run_async=False):
        
        assert isinstance(restart_data, dict)

        if (not self.restart_matches(restart_data.get('role'),
                                     restart_data.get('postgres_version'),
                                     ('restart_pending' in restart_data))):
            return (False, "restart conditions are not satisfied")

        with self._async_executor:
            prev = self._async_executor.schedule('restart')
            if prev is not None:
                return (False, prev + ' already in progress')

            # Make the main loop to think that we were recovering dead postgres. If we fail
            # to start postgres after a specified timeout (see below), we need to remove
            # leader key (if it belong to us) rather than trying to start postgres once again.
            self.recovering = True

        # Now that restart is scheduled we can set timeout for startup, it will get reset
        # once async executor runs and main loop notices PostgreSQL as up.
        timeout = restart_data.get('timeout', self.patroni.config['master_start_timeout'])
        self.set_start_timeout(timeout)

        # For non async cases we want to wait for restart to complete or timeout before returning.
        do_restart = functools.partial(self.state_handler.restart, timeout, self._async_executor.critical_task)
        if self.is_synchronous_mode() and not self.has_lock():
            do_restart = functools.partial(self.while_not_sync_standby, do_restart)

        if run_async:
            self._async_executor.run_async(do_restart)
            return (True, 'restart initiated')
        else:
            res = self._async_executor.run(do_restart)
            if res:
                return (True, 'restarted successfully')
            elif res is None:
                return (False, 'postgres is still starting')
            else:
                return (False, 'restart failed')
265813
private function ensureProcessIsStarted(): void
    {
        if (is_resource($this->process) === false) {
            $this->startProcess();

            $this->handleStartupErrors();
        }
    }
218210
def handle(self):
        ''''''
        try:
            yield self
        except:
            # Requeue the message and raise the original exception
            typ, value, trace = sys.exc_info()
            if not self.processed:
                try:
                    self.req(self.delay())
                except socket.error:
                    self.connection.close()
            raise typ, value, trace
        else:
            if not self.processed:
                try:
                    self.fin()
                except socket.error:
                    self.connection.close()