Greps qstat -e <job_id> for information from the queue.
        :paramsstatus_type: complete, queued, running, error, gone
551477
def _grep_qstat(self, status_type='complete'):
        
        args = "qstat -e {}".format(self.id).split()
        res, _ = call(args)
        if res == '': return False
        res = res.split('\n')[2].split()[4]

        if status_type == 'complete' and res == 'C':
            return True
        elif status_type == 'error' and (res == 'E' or res == 'C'):
            return True
        elif status_type == 'running' and res == 'R':
            return True
        elif status_type == 'queued' and res == 'Q':
            return True
        elif status_type == 'gone' and 'unknown job id' in str(res).lower():
            return True
        else:
            return False
456062
def len(self, queue_name):
        
        try:
            stats = self.conn.stats_tube(queue_name)
        except beanstalkc.CommandFailed as err:
            if err[1] == 'NOT_FOUND':
                return 0

            raise

        return stats.get('current-jobs-ready', 0)
310985
private void dumpStatus(long now) {
    if (now - lastDumpStatusTime < dumpStatusPeriod) {
      return;
    }
    lastDumpStatusTime = now;
    logJobStats(infos.keySet(), TaskType.MAP);
    logJobStats(infos.keySet(), TaskType.REDUCE);
    dumpSpeculationStatus(now);
  }
511543
def backend_status(self, backend='ibmqx4', access_token=None, user_id=None):
        
        if access_token:
            self.req.credential.set_token(access_token)
        if user_id:
            self.req.credential.set_user_id(user_id)
        backend_type = self._check_backend(backend, 'status')
        if not backend_type:
            raise BadBackendError(backend)

        status = self.req.get('/Backends/' + backend_type + '/queue/status',
                              with_token=False)

        ret = {}
        if 'state' in status:
            ret['available'] = bool(status['state'])
        if 'busy' in status:
            ret['busy'] = bool(status['busy'])
        if 'lengthQueue' in status:
            ret['pending_jobs'] = status['lengthQueue']
        
        ret['backend'] = backend_type

        return ret
385553
def status(sec):
    
    if _meta_.prg_bar in ["on", "ON"]:
        syms = ["|", "/", "-", "\\"]
        for sym in syms:
            sys.stdout.write("\b{0}{1}{2}".format(_meta_.color["GREY"], sym,
                                                  _meta_.color["ENDC"]))
            sys.stdout.flush()
            time.sleep(float(sec))
706797
def query(logfile, jobID = None):
    """"""
    
    joblist = logfile.readFromLogfile()
    if jobID and type(jobID) == type(1):
        command = ['qstat', '-j', str(jobID)]
    else:
        command = ['qstat']

    processoutput = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
    output = processoutput[0]
    serror = processoutput[1]
    # Form command
    jobs = {}
    if type(jobID) == type(1):
        if serror.find("Following jobs do not exist") != -1:
            return False
        else:
            return True

    if not output.strip():
        colorprinter.message("No jobs running at present.")
    output = output.strip().split("\n")
    if len(output) > 2:
        for line in output[2:]:
            # We assume that our script names contain no spaces for the parsing below to work
            tokens = line.split()
            jid = int(tokens[0])
            jobstate = tokens[4]

            details = {  "jobid" : jid,
                         "prior" : tokens[1],
                         "name" : tokens[2],
                         "user" : tokens[3],
                         "state" : jobstate,
                         "submit/start at" : "%s %s" % (tokens[5], tokens[6])
                         }
            jataskID = 0
            if jobstate == "r":
                details["queue"] = tokens[7]
                details["slots"] = tokens[8]
            elif jobstate == "qw":
                details["slots"] = tokens[7]
                if len(tokens) >= 9:
                    jataskID = tokens[8]
                    details["ja-task-ID"] = jataskID

            if len(tokens) > 9:
                jataskID = tokens[9]
                details["ja-task-ID"] = jataskID

            jobs[jid] = jobs.get(jid) or {}
            jobs[jid][jataskID] = details
            if joblist.get(jid):
                jobdir = joblist[jid]["Directory"]
                jobtime = joblist[jid]["TimeInSeconds"]
                colorprinter.message("Job %d submitted %d minutes ago. Status: '%s'. Destination directory: %s." % (jid, jobtime / 60, jobstate, jobdir))
            else:
                colorprinter.message("Job %d submitted at %s %s. Status: '%s'. Destination directory unknown." % (jid, tokens[5], tokens[6], jobstate))
        return True
619841
private <Q, R> void recordFinalStatus(JobIdentifier<Q, R> jobId, JobStatus<Q, R> jobStatus) {
        try {
            _jobStatusDAO.updateJobStatus(jobId, jobStatus);
        } catch (Exception e) {
            _log.error("Failed to record final status for job: [id={}, status={}]", jobId, jobStatus.getStatus(), e);
        }
    }
648377
def is_error(self):
        
        qstat = self._grep_qstat('error')
        err = self._grep_status('error')
        if qstat and err:
            return True
        return False
122091
def metrics(self, queue_type=None, queue_id=None):
        
        if queue_id is not None and not is_valid_identifier(queue_id):
            raise BadArgumentException('`queue_id` has an invalid value.')

        if queue_type is not None and not is_valid_identifier(queue_type):
            raise BadArgumentException('`queue_type` has an invalid value.')

        response = {
            'status': 'failure'
        }
        if not queue_type and not queue_id:
            # return global stats.
            # list of active queue types (ready + active)
            active_queue_types = self._r.smembers(
                '%s:active:queue_type' % self._key_prefix)
            ready_queue_types = self._r.smembers(
                '%s:ready:queue_type' % self._key_prefix)
            all_queue_types = active_queue_types | ready_queue_types
            # global rates for past 10 minutes
            timestamp = str(generate_epoch())
            keys = [
                self._key_prefix
            ]
            args = [
                timestamp
            ]

            enqueue_details, dequeue_details = self._lua_metrics(
                keys=keys, args=args)

            enqueue_counts = {}
            dequeue_counts = {}
            # the length of enqueue & dequeue details are always same.
            for i in xrange(0, len(enqueue_details), 2):
                enqueue_counts[str(enqueue_details[i])] = int(
                    enqueue_details[i + 1] or 0)
                dequeue_counts[str(dequeue_details[i])] = int(
                    dequeue_details[i + 1] or 0)

            response.update({
                'status': 'success',
                'queue_types': list(all_queue_types),
                'enqueue_counts': enqueue_counts,
                'dequeue_counts': dequeue_counts
            })
            return response
        elif queue_type and not queue_id:
            # return list of queue_ids.
            # get data from two sorted sets in a transaction
            pipe = self._r.pipeline()
            pipe.zrange('%s:%s' % (self._key_prefix, queue_type), 0, -1)
            pipe.zrange('%s:%s:active' % (self._key_prefix, queue_type), 0, -1)
            ready_queues, active_queues = pipe.execute()
            # extract the queue_ids from the queue_id:job_id string
            active_queues = [i.split(':')[0] for i in active_queues]
            all_queue_set = set(ready_queues) | set(active_queues)
            response.update({
                'status': 'success',
                'queue_ids': list(all_queue_set)
            })
            return response
        elif queue_type and queue_id:
            # return specific details.
            active_queue_types = self._r.smembers(
                '%s:active:queue_type' % self._key_prefix)
            ready_queue_types = self._r.smembers(
                '%s:ready:queue_type' % self._key_prefix)
            all_queue_types = active_queue_types | ready_queue_types
            # queue specific rates for past 10 minutes
            timestamp = str(generate_epoch())
            keys = [
                '%s:%s:%s' % (self._key_prefix, queue_type, queue_id)
            ]
            args = [
                timestamp
            ]

            enqueue_details, dequeue_details = self._lua_metrics(
                keys=keys, args=args)

            enqueue_counts = {}
            dequeue_counts = {}
            # the length of enqueue & dequeue details are always same.
            for i in xrange(0, len(enqueue_details), 2):
                enqueue_counts[str(enqueue_details[i])] = int(
                    enqueue_details[i + 1] or 0)
                dequeue_counts[str(dequeue_details[i])] = int(
                    dequeue_details[i + 1] or 0)

            # get the queue length for the job queue
            queue_length = self._r.llen('%s:%s:%s' % (
                self._key_prefix, queue_type, queue_id))

            response.update({
                'status': 'success',
                'queue_length': int(queue_length),
                'enqueue_counts': enqueue_counts,
                'dequeue_counts': dequeue_counts
            })
            return response
        elif not queue_type and queue_id:
            raise BadArgumentException(
                '`queue_id` should be accompanied by `queue_type`.')

        return response
113396
def _PrintTasksStatus(self, processing_status):
    
    if processing_status and processing_status.tasks_status:
      tasks_status = processing_status.tasks_status

      table_view = views.CLITabularTableView(
          column_names=['Tasks:', 'Queued', 'Processing', 'Merging',
                        'Abandoned', 'Total'],
          column_sizes=[15, 7, 15, 15, 15, 0])

      table_view.AddRow([
          '', tasks_status.number_of_queued_tasks,
          tasks_status.number_of_tasks_processing,
          tasks_status.number_of_tasks_pending_merge,
          tasks_status.number_of_abandoned_tasks,
          tasks_status.total_number_of_tasks])

      self._output_writer.Write('\n')
      table_view.Write(self._output_writer)
468985
def update_job_status(self, checker_func):
        """"""
        njobs = len(self.cache.keys())
        status_vect = np.zeros((8), int)
        sys.stdout.write("Updating status of %i jobs: " % njobs)
        sys.stdout.flush()
        for i, key in enumerate(self.cache.keys()):
            if i % 200 == 0:
                sys.stdout.write('.')
                sys.stdout.flush()
            job_details = self.cache[key]
            if job_details.status in [JobStatus.pending, JobStatus.running]:
                if checker_func:
                    job_details.check_status_logfile(checker_func)
            job_details.update_table_row(self._table, job_details.dbkey - 1)
            status_vect[job_details.status] += 1

        sys.stdout.write("!\n")
        sys.stdout.flush()
        sys.stdout.write("Summary:\n")
        sys.stdout.write("  Unknown:   %i\n" % status_vect[JobStatus.unknown])
        sys.stdout.write("  Not Ready: %i\n" %
                         status_vect[JobStatus.not_ready])
        sys.stdout.write("  Ready:     %i\n" % status_vect[JobStatus.ready])
        sys.stdout.write("  Pending:   %i\n" % status_vect[JobStatus.pending])
        sys.stdout.write("  Running:   %i\n" % status_vect[JobStatus.running])
        sys.stdout.write("  Done:      %i\n" % status_vect[JobStatus.done])
        sys.stdout.write("  Failed:    %i\n" % status_vect[JobStatus.failed])
        sys.stdout.write("  Partial:   %i\n" %
                         status_vect[JobStatus.partial_failed])
467785
def queue_stats(self, queue, tags):
        
        for mname, pymqi_value in iteritems(metrics.queue_metrics()):
            try:
                mname = '{}.queue.{}'.format(self.METRIC_PREFIX, mname)
                m = queue.inquire(pymqi_value)
                self.gauge(mname, m, tags=tags)
            except pymqi.Error as e:
                self.warning("Error getting queue stats for {}: {}".format(queue, e))

        for mname, func in iteritems(metrics.queue_metrics_functions()):
            try:
                mname = '{}.queue.{}'.format(self.METRIC_PREFIX, mname)
                m = func(queue)
                self.gauge(mname, m, tags=tags)
            except pymqi.Error as e:
                self.warning("Error getting queue stats for {}: {}".format(queue, e))
126344
def _dump_queue_stats(self):
        '''
        
        '''
        extras = {}
        keys = self.redis_conn.keys('*:*:queue')
        total_backlog = 0
        for key in keys:
            elements = key.split(":")
            spider = elements[0]
            domain = elements[1]
            spider = 'queue_' + spider

            if spider not in extras:
                extras[spider] = {}
                extras[spider]['spider_backlog'] = 0
                extras[spider]['num_domains'] = 0

            count = self.redis_conn.zcard(key)
            total_backlog += count
            extras[spider]['spider_backlog'] += count
            extras[spider]['num_domains'] += 1

        extras['total_backlog'] = total_backlog

        if not self.logger.json:
            self.logger.info('Queue Stats Dump:\n{0}'.format(
                    json.dumps(extras, indent=4, sort_keys=True)))
        else:
            self.logger.info('Queue Stats Dump', extra=extras)
1324507
def _job_queue_empty(self):
        

        self._iteratively_clean_pending_exits()

        while self._pending_jobs:
            # We don't have any exits remaining. Let's pop out a pending exit
            pending_job = self._get_one_pending_job()
            if pending_job is None:
                continue

            self._insert_job(pending_job)
            self._register_analysis_job(pending_job.func_addr, pending_job)
            break
467826
def get_output_content(job_id, max_size=1024, conn=None):
    
    content = None
    if RBO.index_list().contains(IDX_OUTPUT_JOB_ID).run(conn):
        # NEW
        check_status = RBO.get_all(job_id, index=IDX_OUTPUT_JOB_ID).run(conn)
    else:
        # OLD
        check_status = RBO.filter({OUTPUTJOB_FIELD: {ID_FIELD: job_id}}).run(conn)
    for status_item in check_status:
        content = _truncate_output_content_if_required(status_item, max_size)
    return content
1351954
def clear_queue(self, queue_type=None, queue_id=None, purge_all=False):
        
        if queue_id is None or not is_valid_identifier(queue_id):
            raise BadArgumentException('`queue_id` has an invalid value.')

        if queue_type is None or not is_valid_identifier(queue_type):
            raise BadArgumentException('`queue_type` has an invalid value.')

        response = {
            'status': 'Failure',
            'message': 'No queued calls found'
            }
        # remove from the primary sorted set
        primary_set = '{}:{}'.format(self._key_prefix, queue_type)
        queued_status = self._r.zrem(primary_set, queue_id)
        if queued_status:
            response.update({'status': 'Success', 
                    'message': 'Successfully removed all queued calls'})
        # do a full cleanup of reources
        # although this is not necessary as we don't remove resources 
        # while dequeue operation
        job_queue_list = '{}:{}:{}'.format(self._key_prefix, queue_type, queue_id)
        if queued_status and purge_all:
            job_list = self._r.lrange(job_queue_list, 0, -1)
            pipe = self._r.pipeline()
            # clear the payload data for job_uuid
            for job_uuid in job_list:
                if job_uuid is None:
                    continue
                payload_set = '{}:payload'.format(self._key_prefix)
                job_payload_key = '{}:{}:{}'.format(queue_type, queue_id, job_uuid)
                pipe.hdel(payload_set, job_payload_key)
            # clear jobrequest interval
            interval_set = '{}:interval'.format(self._key_prefix)
            job_interval_key = '{}:{}'.format(queue_type, queue_id)
            pipe.hdel(interval_set, job_interval_key)
            # clear job_queue_list
            pipe.delete(job_queue_list)
            pipe.execute()
            response.update({'status': 'Success', 
                    'message': 'Successfully removed all queued calls and purged related resources'})
        else:
            # always delete the job queue list
            self._r.delete(job_queue_list)
        return response
1027998
def qstat(self, queue_name, return_dict=False):
        
        rtn = self.execute_command('QSTAT', queue_name)

        if return_dict:
            grouped = self._grouper(rtn, 2)
            rtn = dict((a, b) for a, b in grouped)

        return rtn