Delete jobs stalled in the running state for too long

        timeout_last_refresh, int: number of seconds
969199
def delete_running(self, timeout_last_refresh=0, dry_run=False):
        
        running_all = self.handle.jobs_running()
        running_timeout = [job for job in running_all
                           if coarse_utcnow() > job["refresh_time"] +
                           timedelta(seconds=timeout_last_refresh)]
        if len(running_timeout) == 0:
            # Nothing to stop
            self.refresh_tids(None)
            return None

        if dry_run:
            logger.warning("Dry run. Not removing anything.")

        logger.info("Removing {0}/{1} running jobs. # all jobs: {2} ".
                    format(len(running_timeout), len(running_all), len(self)))

        now = coarse_utcnow()
        logger.info("Current utc time: {0}".format(now))
        logger.info("Time horizont: {0}".format(now - timedelta(seconds=timeout_last_refresh)))
        for job in running_timeout:
            logger.info("Removing job: ")
            pjob = job.to_dict()
            del pjob["misc"]  # ignore misc when printing
            logger.info(pprint.pformat(pjob))
            if not dry_run:
                self.handle.delete(job)
                logger.info("Job deleted")
        self.refresh_tids(None)
1166966
def poll_ignore_interrupts(fds, timeout=None):
    ''''''

    if timeout is not None:
        end_time = time.time() + timeout

    poller = select.poll()
    for fd in fds:
        poller.register(fd, select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR)

    while True:
        try:
            timeout_ms = None if timeout is None else timeout * 1000
            results = poller.poll(timeout_ms)
            return [afd for afd, _ in results]
        except InterruptedError:
            err = sys.exc_info()[1]
            if err.args[0] == errno.EINTR:
                # if we loop back we have to subtract the
                # amount of time we already waited.
                if timeout is not None:
                    timeout = end_time - time.time()
                    if timeout < 0:
                        return []
            else:
                # something else caused the select.error, so
                # this actually is an exception.
                raise
137883
public synchronized void cleanWaitTaskQueue() {

        for (ParallelTask task : waitQ) {
            task.setState(ParallelTaskState.COMPLETED_WITH_ERROR);
            task.getTaskErrorMetas().add(
                    new TaskErrorMeta(TaskErrorType.USER_CANCELED, "NA"));
            logger.info(
                    "task {} removed from wait q. This task has been marked as USER CANCELED.",
                    task.getTaskId());

        }

        waitQ.clear();
    }
292382
def _refresh_state(self):
    
    # TODO(gram): should we put a choke on refreshes? E.g. if the last call was less than
    # a second ago should we return the cached value?
    if self._is_complete:
      return

    try:
      response = self._api.jobs_get(self._job_id)
    except Exception as e:
      raise e

    if 'status' in response:
      status = response['status']
      if 'state' in status and status['state'] == 'DONE':
        self._end_time = datetime.datetime.utcnow()
        self._is_complete = True
        self._process_job_status(status)

    if 'statistics' in response:
      statistics = response['statistics']
      start_time = statistics.get('creationTime', None)
      end_time = statistics.get('endTime', None)
      if start_time and end_time and end_time >= start_time:
        self._start_time = datetime.datetime.fromtimestamp(float(start_time) / 1000.0)
        self._end_time = datetime.datetime.fromtimestamp(float(end_time) / 1000.0)
785047
def clear(self):
    """"""
    while self.current or self.idlers or self.queue or self.rpcs:
      current = self.current
      idlers = self.idlers
      queue = self.queue
      rpcs = self.rpcs
      _logging_debug('Clearing stale EventLoop instance...')
      if current:
        _logging_debug('  current = %s', current)
      if idlers:
        _logging_debug('  idlers = %s', idlers)
      if queue:
        _logging_debug('  queue = %s', queue)
      if rpcs:
        _logging_debug('  rpcs = %s', rpcs)
      self.__init__()
      current.clear()
      idlers.clear()
      queue[:] = []
      rpcs.clear()
      _logging_debug('Cleared')
821433
def stop_refresh(self):
        
        self.logger.debug("stopping timed refresh")
        self.rf_flags['done'] = True
        self.rf_timer.clear()
1061170
synchronized int removeOldConnections() {
        long now = System.currentTimeMillis();
        long expirationTime = now - idleTransportTimeout.millis();

        List<PooledTransport> removeFromIdle = new ArrayList<PooledTransport>();
        for (Map.Entry<PooledTransport, Long> idleEntry : idle.entrySet()) {
            long lastUsed = idleEntry.getValue();
            if (lastUsed < expirationTime) {
                PooledTransport removed = idleEntry.getKey();
                if (log.isTraceEnabled()) {
                    log.trace("Expiring idle transport for job [" + jobPoolingKey + "], transport: ["
                            + removed.toString() + "]. Last used [" + new TimeValue(now-lastUsed) + "] ago. Expired ["
                            + idleTransportTimeout + "] ago.");
                }
                release(removed);
                removeFromIdle.add(removed);
            }
        }

        for (PooledTransport toRemove : removeFromIdle) {
            idle.remove(toRemove);
        }

        return idle.size() + leased.size();
    }
180592
def DeleteOldRuns(self, job, cutoff_timestamp=None, token=None):
    """"""
    if cutoff_timestamp is None:
      raise ValueError("cutoff_timestamp can't be None")

    child_flows = list(job.ListChildren(age=cutoff_timestamp))
    with queue_manager.QueueManager(token=token) as queuemanager:
      queuemanager.MultiDestroyFlowStates(child_flows)

    aff4.FACTORY.MultiDelete(child_flows, token=token)
    return len(child_flows)
197138
def halt_running():
    ''''''
    vultr = Vultr(API_KEY)

    try:
            serverList = vultr.server.list()
            #logging.info('Listing servers:\n%s', dumps(
            #serverList, indent=2
        #))
    except VultrError as ex:
        logging.error('VultrError: %s', ex)

    for serverID in serverList:
        if serverList[serverID]['power_status'] == 'running':
            logging.info(serverList[serverID]['label'] + " will be gracefully shutdown.")
            vultr.server.halt(serverID)
940487
def delete(args):
  """"""
  jm = setup(args)
  # first, stop the jobs if they are running in the grid
  if not args.local and 'executing' in args.status:
    stop(args)
  # then, delete them from the database
  jm.delete(job_ids=get_ids(args.job_ids), array_ids=get_ids(args.array_ids), delete_logs=not args.keep_logs, delete_log_dir=not args.keep_log_dir, status=args.status)
398791
async def _clean_shutdown(self):
        """"""

        # Cleanly stop any other outstanding tasks not associated with tiles
        remaining_tasks = []
        for task in self._tasks.get(None, []):
            self._logger.debug("Cancelling task at shutdown %s", task)
            task.cancel()

            remaining_tasks.append(task)

        asyncio.gather(*remaining_tasks, return_exceptions=True)

        if len(remaining_tasks) > 0:
            del self._tasks[None]

        # Shutdown tasks associated with each tile
        remaining_tasks = []

        for address in sorted(self._tasks, reverse=True):
            if address is None:
                continue

            self._logger.debug("Shutting down tasks for tile at %d", address)
            for task in self._tasks.get(address, []):
                task.cancel()
                remaining_tasks.append(task)

        asyncio.gather(*remaining_tasks, return_exceptions=True)

        await self._rpc_queue.stop()

        self._loop.stop()
34989
void performCleanUp(@Nullable Runnable task) {
    evictionLock.lock();
    try {
      maintenance(task);
    } finally {
      evictionLock.unlock();
    }
    if ((drainStatus() == REQUIRED) && (executor == ForkJoinPool.commonPool())) {
      scheduleDrainBuffers();
    }
  }
1272023
def _AbortJoin(self, timeout=None):
    
    for pid, process in iter(self._processes_per_pid.items()):
      logger.debug('Waiting for process: {0:s} (PID: {1:d}).'.format(
          process.name, pid))
      process.join(timeout=timeout)
      if not process.is_alive():
        logger.debug('Process {0:s} (PID: {1:d}) stopped.'.format(
            process.name, pid))
461989
def cancel(self, job_ids):
        ''' 
        '''

        job_id_list = ' '.join(job_ids)
        retcode, stdout, stderr = self.channel.execute_wait("qdel {0}".format(job_id_list), 3)
        rets = None
        if retcode == 0:
            for jid in job_ids:
                self.resources[jid]['status'] = translate_table['E']  # Setting state to exiting
            rets = [True for i in job_ids]
        else:
            rets = [False for i in job_ids]

        return rets
691680
def cancel(self, job_ids):
        ''' 
        '''

        for job in job_ids:
            logger.debug("Terminating job/proc_id : {0}".format(job))
            # Here we are assuming that for local, the job_ids are the process id's
            proc = self.resources[job]['proc']
            os.killpg(os.getpgid(proc.pid), signal.SIGTERM)
            self.resources[job]['status'] = 'CANCELLED'
        rets = [True for i in job_ids]

        return rets
1129170
def _cleanup(self):  # pragma: no cover
        
        need_to_delete = []  # can't delete from a dict while iterating
        with self.attr_lock:
            now_time = time()
            for path in self.cache:
                if now_time - self.attr[path][TIMESTAMP_KEY] >= MAX_CACHE_TIME:
                    need_to_delete.append(path)
            for path in need_to_delete:
                del self.attr[path]
                del self.cache[path]
934600
public synchronized void setFlushInterval(long delay, long seconds) {
        if (m_flush != null) {
            m_flush.cancel(false);
            m_flush = null;
        }
        if (seconds > 0) {
            m_flush = m_ses.scheduleAtFixedRate(new Runnable() {
                @Override
                public void run() {
                    try {
                        flush();
                    } catch (Exception e) {
                        loaderLog.error("Failed to flush loader buffer, some tuples may not be inserted.", e);
                    }
                }
            }, delay, seconds, TimeUnit.SECONDS);
        }
    }