Return a dict of queues in cluster and servers running them
546051
def clusterQueues(self):
        
        servers = yield self.getClusterServers()

        queues = {}

        for sname in servers:
            qs = yield self.get('rhumba.server.%s.queues' % sname)
            uuid = yield self.get('rhumba.server.%s.uuid' % sname)
       
            qs = json.loads(qs)

            for q in qs:
                if q not in queues:
                    queues[q] = []

                queues[q].append({'host': sname, 'uuid': uuid})

        defer.returnValue(queues)
8446
def list_nodes_min(conn=None, call=None):
    '''
    

    '''
    if call == 'action':
        raise SaltCloudSystemExit(
            'The list_nodes_min function must be called with -f or --function.'
        )
    if conn is None:
        conn = get_conn()
    ret = {}
    for node in conn.list_servers(bare=True):
        ret[node.name] = {'id': node.id, 'state': node.status}
    return ret
1265710
def queues(self, page=None, per_page=None, previous=None, prefix=None):
        
        options = {}
        if page is not None:
            raise Exception('page param is deprecated!')
        if per_page is not None:
            options['per_page'] = per_page
        if previous is not None:
            options['previous'] = previous
        if prefix is not None:
            options['prefix'] = prefix

        query = urlencode(options)
        url = 'queues'
        if query != '':
            url = "%s?%s" % (url, query)
        result = self.client.get(url)

        return [queue['name'] for queue in result['body']['queues']]
997009
def to_dict(self):
        
        return {
            'name': self.name,
            'broker': self.broker.to_dict(),
            'pid': self.pid,
            'process_pids': self.process_pids,
            'concurrency': self.concurrency,
            'job_count': self.job_count,
            'queues': [q.to_dict() for q in self.queues]
        }
538235
def _get_index_servers(self):
        """"""
        idx_srvs = []
        if 'index-servers' in self.conf.options('distutils'):
            idx = self.conf.get('distutils', 'index-servers')
            idx_srvs = [srv.strip() for srv in idx.split('\n') if srv.strip()]
        return idx_srvs
443130
def list_servers(backend, socket=DEFAULT_SOCKET_URL, objectify=False):
    '''
    
    '''
    ha_conn = _get_conn(socket)
    ha_cmd = haproxy.cmds.listServers(backend=backend)
    return ha_conn.sendCmd(ha_cmd, objectify=objectify)
1133960
def all_known(cls, sources=None, prefixes=None):
        

        sources = sources or ("config", "jobs", "raw_subqueues")

        queues = set()

        if "config" in sources and not prefixes:
            # Some queues are explicitly declared in the config (including all root raw queues)
            cfg = context.get_current_config()

            queues_from_config = [
                t.get("queue")
                for t in (cfg.get("tasks") or {}).values()
                if t.get("queue")
            ]

            queues_from_config += Queue.get_queues_config().keys()

            queues_from_config += [
                t.get("retry_queue")
                for t in Queue.get_queues_config().values()
                if t.get("retry_queue")
            ]

            queues |= set(queues_from_config)

        if "jobs" in sources:

            # This will get all queues from mongodb, including those where we have only non-queued jobs
            for q in context.connections.mongodb_jobs.mrq_jobs.distinct("queue"):
                if prefixes and not any(q.startswith(p) for p in prefixes):
                    continue
                queues.add(q)

        if "raw_subqueues" in sources:
            for q in Queue.get_queues_config():
                if prefixes and not any(q + "/" == p for p in prefixes):
                    continue
                queue_obj = Queue(q)
                if queue_obj.is_raw and queue_obj.has_subqueues:
                    # TODO: optimize this with a single SUNION on all keys
                    queues |= queue_obj.get_known_subqueues()

        return queues
1317157
def _all_queue_names(self):
        
        queues = set()
        endpoints = self.config.get('endpoints')
        for e in endpoints:
            for q in endpoints[e]['queues']:
                queues.add(q)
        return sorted(queues)
844046
def list_hosts(kwargs=None, call=None):
    '''
    
    '''
    if call != 'function':
        raise SaltCloudSystemExit(
            'The list_hosts function must be called with '
            '-f or --function.'
        )

    return {'Hosts': salt.utils.vmware.list_hosts(_get_si())}
636965
def clusterStatus(self):
        
        servers = yield self.getClusterServers()

        d = {
            'workers': {},
            'crons': {},
            'queues': {}
        }

        now = time.time()

        reverse_map = {}

        for sname in servers:
            last = yield self.get('rhumba.server.%s.heartbeat' % sname)
            status = yield self.get('rhumba.server.%s.status' % sname)
            uuid = yield self.get('rhumba.server.%s.uuid' % sname)

            reverse_map[uuid] = sname

            if not last:
                last = 0

            last = float(last)

            if (status == 'ready') and (now - last > 5):
                status = 'offline'

            if not sname in d['workers']:
                d['workers'][sname] = []

            d['workers'][sname].append({
                'lastseen': last,
                'status': status,
                'id': uuid
            })

        # Crons
        crons = yield self.keys('rhumba\.crons\.*')

        for key in crons:
            segments = key.split('.')

            queue = segments[2]
            if queue not in d['crons']:
                d['crons'][queue] = {'methods': {}}

            if len(segments)==4:
                last = yield self.get(key)
                d['crons'][queue]['methods'][segments[3]] = float(last)
            else:
                uid = yield self.get(key)
                d['crons'][queue]['master'] = '%s:%s' % (uid, reverse_map[uid])

        # Queues
        queue_keys = yield self.keys('rhumba.qstats.*')

        for key in queue_keys:
            qname = key.split('.')[2]
            if qname not in d['queues']:
                qlen = yield self.queueSize(qname)

                stats = yield self.getQueueMessageStats(qname)

                d['queues'][qname] = {
                    'waiting': qlen,
                    'messages': stats
                }

        defer.returnValue(d)
575994
def queue(self):
        """"""
        with self.connection_pool.acquire(block=True) as conn:
            return Q(
                self.routing_key,
                exchange=self.exchange,
                routing_key=self.routing_key
            )(conn)
10732
def get_list_cluster_admins(self):
        """"""
        response = self.request(
            url="cluster_admins",
            method='GET',
            expected_response_code=200
        )

        return response.json()
850146
def as_dict(self):
        
        if self.has_omp:
            raise NotImplementedError('as_dict method of QueueAdapter not yet implemented when OpenMP is activated')
        return {'@module': self.__class__.__module__,
                '@class': self.__class__.__name__,
                'priority': self.priority,
                'hardware': self.hw.as_dict(),
                'queue': {'qtype': self.QTYPE,
                          'qname': self._qname,
                          'qnodes': self.qnodes,
                          'qparams': self._qparams},
                'limits': {'timelimit_hard': self._timelimit_hard,
                           'timelimit': self._timelimit,
                           'min_cores': self.min_cores,
                           'max_cores': self.max_cores,
                           'min_mem_per_proc': self.min_mem_per_proc,
                           'max_mem_per_proc': self.max_mem_per_proc,
                           'memory_policy': self.memory_policy
                           },
                'job': {},
                'mpi_procs': self._mpi_procs,
                'mem_per_proc': self._mem_per_proc,
                'master_mem_overhead': self._master_mem_overhead
                }
408851
def _get_tasks_by_queue(self):
        """"""
        task_map = {}
        _checker = None

        # Ask the persistence engine for an Async to use for checking if the
        # context is complete.
        if self._persistence_engine:
            _checker = self._persistence_engine.context_completion_checker

        for async in self._tasks:
            queue = async.get_queue()
            if _checker:
                async.update_options(_context_checker=_checker)

            task = async.to_task()
            task_map.setdefault(queue, []).append(task)

        return task_map
1146073
def get_all_queues(self, prefix=''):
        
        params = {}
        if prefix:
            params['QueueNamePrefix'] = prefix
        return self.get_list('ListQueues', params, [('QueueUrl', Queue)])
126344
def _dump_queue_stats(self):
        '''
        
        '''
        extras = {}
        keys = self.redis_conn.keys('*:*:queue')
        total_backlog = 0
        for key in keys:
            elements = key.split(":")
            spider = elements[0]
            domain = elements[1]
            spider = 'queue_' + spider

            if spider not in extras:
                extras[spider] = {}
                extras[spider]['spider_backlog'] = 0
                extras[spider]['num_domains'] = 0

            count = self.redis_conn.zcard(key)
            total_backlog += count
            extras[spider]['spider_backlog'] += count
            extras[spider]['num_domains'] += 1

        extras['total_backlog'] = total_backlog

        if not self.logger.json:
            self.logger.info('Queue Stats Dump:\n{0}'.format(
                    json.dumps(extras, indent=4, sort_keys=True)))
        else:
            self.logger.info('Queue Stats Dump', extra=extras)
912985
def list_(prefix='', region=None, key=None, keyid=None, profile=None):
    '''
    
    '''
    conn = _get_conn(region=region, key=key, keyid=keyid, profile=profile)

    def extract_name(queue_url):
        # Note: this logic taken from boto, so should be safe
        return _urlparse(queue_url).path.split('/')[2]

    try:
        r = conn.list_queues(QueueNamePrefix=prefix)
        # The 'QueueUrls' attribute is missing if there are no queues
        urls = r.get('QueueUrls', [])
        return {'result': [extract_name(url) for url in urls]}
    except botocore.exceptions.ClientError as e:
        return {'error': __utils__['boto3.get_error'](e)}