Return the total number of bytes of requested, unprocessed replicas.
483949
def get_total_size_of_queued_replicas():
    """"""
    return (
        d1_gmn.app.models.ReplicationQueue.objects.filter(
            local_replica__info__status__status='queued'
        ).aggregate(Sum('size'))['size__sum']
        or 0
    )
844396
def display_replica_imbalance(cluster_topologies):
    
    assert cluster_topologies

    rg_ids = list(next(six.itervalues(cluster_topologies)).rgs.keys())
    assert all(
        set(rg_ids) == set(cluster_topology.rgs.keys())
        for cluster_topology in six.itervalues(cluster_topologies)
    )

    rg_imbalances = [
        stats.get_replication_group_imbalance_stats(
            list(cluster_topology.rgs.values()),
            list(cluster_topology.partitions.values()),
        )
        for cluster_topology in six.itervalues(cluster_topologies)
    ]

    _display_table_title_multicolumn(
        'Extra Replica Count',
        'Replication Group',
        rg_ids,
        list(cluster_topologies.keys()),
        [
            [erc[rg_id] for rg_id in rg_ids]
            for _, erc in rg_imbalances
        ],
    )

    for name, imbalance in zip(
            six.iterkeys(cluster_topologies),
            (imbalance for imbalance, _ in rg_imbalances)
    ):
        print(
            '\n'
            '{name}'
            'Total extra replica count: {imbalance}'
            .format(
                name='' if len(cluster_topologies) == 1 else name + '\n',
                imbalance=imbalance,
            )
        )
630436
def metrics(self):
        
        masterThrp, backupThrp = self.getThroughputs(self.instances.masterId)
        r = self.instance_throughput_ratio(self.instances.masterId)
        m = [
            ("{} Monitor metrics:".format(self), None),
            ("Delta", self.Delta),
            ("Lambda", self.Lambda),
            ("Omega", self.Omega),
            ("instances started", self.instances.started),
            ("ordered request counts",
             {i: r[0] for i, r in self.numOrderedRequests.items()}),
            ("ordered request durations",
             {i: r[1] for i, r in self.numOrderedRequests.items()}),
            ("master request latencies", self.masterReqLatencies),
            ("client avg request latencies", {i: self.getLatency(i)
                                              for i in self.instances.ids}),
            ("throughput", {i: self.getThroughput(i)
                            for i in self.instances.ids}),
            ("master throughput", masterThrp),
            ("total requests", self.totalRequests),
            ("avg backup throughput", backupThrp),
            ("master throughput ratio", r)]
        return m
948581
def _prepare_output(partitions, unavailable_brokers, verbose):
    """"""
    partitions_count = len(partitions)
    out = {}
    out['raw'] = {
        'replica_unavailability_count': partitions_count,
    }

    if partitions_count == 0:
        out['message'] = 'All replicas available for communication.'
    else:
        out['message'] = "{replica_unavailability} replicas unavailable for communication. " \
            "Unavailable Brokers: {unavailable_brokers}".format(
            replica_unavailability=partitions_count,
            unavailable_brokers=', '.join([str(e) for e in unavailable_brokers]),
        )
        if verbose:
            lines = (
                '{}:{}'.format(topic, partition)
                for (topic, partition) in partitions
            )
            out['verbose'] = "Partitions:\n" + "\n".join(lines)

    if verbose:
        out['raw']['partitions'] = [
            {'topic': topic, 'partition': partition}
            for (topic, partition) in partitions
        ]

    return out
838672
def count(self) -> int:
        ''''''
        counter = 0

        for pool in self._host_pools.values():
            counter += pool.count()

        return counter
542317
def service_replicas_outbox(self, limit: int = None) -> int:
        
        # TODO: rewrite this using Router

        num_processed = 0
        for message in self.replicas.get_output(limit):
            num_processed += 1
            if isinstance(message, (PrePrepare, Prepare, Commit, Checkpoint)):
                self.send(message)
            elif isinstance(message, Ordered):
                self.try_processing_ordered(message)
            elif isinstance(message, tuple) and isinstance(message[1], Reject):
                with self.metrics.measure_time(MetricsName.NODE_SEND_REJECT_TIME):
                    digest, reject = message
                    result_reject = Reject(
                        reject.identifier,
                        reject.reqId,
                        self.reasonForClientFromException(
                            reject.reason))
                    # TODO: What the case when reqKey will be not in requestSender dict
                    if digest in self.requestSender:
                        self.transmitToClient(result_reject, self.requestSender[digest])
                        self.doneProcessingReq(digest)
            elif isinstance(message, Exception):
                self.processEscalatedException(message)
            else:
                # TODO: should not this raise exception?
                logger.error("Received msg {} and don't "
                             "know how to handle it".format(message))
        return num_processed
637252
def complete_offset_upload(self, chunk_num):
        # type: (Descriptor, int) -> None
        
        with self._meta_lock:
            self._outstanding_ops -= 1
            # save resume state
            if self.is_resumable:
                # only set resumable completed if all replicas for this
                # chunk are complete
                if blobxfer.util.is_not_empty(self._ase.replica_targets):
                    if chunk_num not in self._replica_counters:
                        # start counter at -1 since we need 1 "extra" for the
                        # primary in addition to the replica targets
                        self._replica_counters[chunk_num] = -1
                    self._replica_counters[chunk_num] += 1
                    if (self._replica_counters[chunk_num] !=
                            len(self._ase.replica_targets)):
                        return
                    else:
                        self._replica_counters.pop(chunk_num)
                self._completed_chunks.set(True, chunk_num)
                completed = self._outstanding_ops == 0
                if not completed and self.must_compute_md5:
                    last_consecutive = (
                        self._completed_chunks.find('0b0')[0] - 1
                    )
                    md5digest = self._md5_cache[last_consecutive]
                else:
                    md5digest = None
                self._resume_mgr.add_or_update_record(
                    self.local_path.absolute_path, self._ase, self._chunk_size,
                    self._total_chunks, self._completed_chunks.int, completed,
                    md5digest,
                )
                # prune md5 cache
                if self.must_compute_md5:
                    if completed:
                        self._md5_cache.clear()
                    elif (len(self._md5_cache) >
                          _MD5_CACHE_RESUME_ENTRIES_GC_THRESHOLD):
                        mkeys = sorted(list(self._md5_cache.keys()))
                        for key in mkeys:
                            if key >= last_consecutive:
                                break
                            self._md5_cache.pop(key)
824490
def gather_data(registry):
    """"""

    # Get the host name of the machine
    host = socket.gethostname()

    # Create our collectors
    ram_metric = Gauge("memory_usage_bytes", "Memory usage in bytes.",
                       {'host': host})
    cpu_metric = Gauge("cpu_usage_percent", "CPU usage percent.",
                       {'host': host})

    # register the metric collectors
    registry.register(ram_metric)
    registry.register(cpu_metric)

    # Start gathering metrics every second
    while True:
        time.sleep(1)

        # Add ram metrics
        ram = psutil.virtual_memory()
        swap = psutil.swap_memory()

        ram_metric.set({'type': "virtual", }, ram.used)
        ram_metric.set({'type': "virtual", 'status': "cached"}, ram.cached)
        ram_metric.set({'type': "swap"}, swap.used)

        # Add cpu metrics
        for c, p in enumerate(psutil.cpu_percent(interval=1, percpu=True)):
            cpu_metric.set({'core': c}, p)
617509
def pending(self, partitions=None):
        
        if partitions is None:
            partitions = self.offsets.keys()

        total = 0
        reqs = []

        for partition in partitions:
            reqs.append(OffsetRequestPayload(self.topic, partition, -1, 1))

        resps = self.client.send_offset_request(reqs)
        for resp in resps:
            partition = resp.partition
            pending = resp.offsets[0]
            offset = self.offsets[partition]
            total += pending - offset

        return total
860357
def get_remote_content_len(self, remote, headers=None):
        

        if headers is None:
            headers = self._get_default_request_headers()

        req = urllib.request.Request(remote, headers=headers)

        try:
            response = urllib.request.urlopen(req)
            resp_header = response.info()
            byte_size = resp_header.get('Content-length')
        except OSError as err:
            byte_size = None
            LOG.error(err)

        return byte_size
1305972
private byte[] replicas( ) {
    byte[] r = _replicas;
    if( r != null ) return r;
    byte[] nr = new byte[H2O.CLOUD.size()+1/*1-based numbering*/+10/*limit of 10 clients*/];
    if( REPLICAS_UPDATER.compareAndSet(this,null,nr) ) return nr;
    r = _replicas/*read again, since CAS failed must be set now*/;
    assert r!= null;
    return r;
  }
1077673
def nbytes_stored(self):
        """"""
        m = getsize(self._store, self._path)
        if self._chunk_store is None:
            return m
        else:
            n = getsize(self._chunk_store, self._path)
            if m < 0 or n < 0:
                return -1
            else:
                return m + n
1369657
def collect_instance(self, nick, host, port, unix_socket, auth):
        

        # Connect to redis and get the info
        info = self._get_info(host, port, unix_socket, auth)
        if info is None:
            return

        # The structure should include the port for multiple instances per
        # server
        data = dict()

        # Role needs to be handled outside the the _KEYS dict
        # since the value is a string, not a int / float
        # Also, master_sync_in_progress is only available if the
        # redis instance is a slave, so default it here so that
        # the metric is cleared if the instance flips from slave
        # to master
        if 'role' in info:
            if info['role'] == "master":
                data['replication.master'] = 1
                data['replication.master_sync_in_progress'] = 0
            else:
                data['replication.master'] = 0

        # Connect to redis and get the maxmemory config value
        # Then calculate the % maxmemory of memory used
        maxmemory_config = self._get_config(host, port, unix_socket, auth,
                                            'maxmemory')
        if maxmemory_config and 'maxmemory' in maxmemory_config.keys():
            maxmemory = float(maxmemory_config['maxmemory'])

            # Only report % used if maxmemory is a non zero value
            if maxmemory == 0:
                maxmemory_percent = 0.0
            else:
                maxmemory_percent = info['used_memory'] / maxmemory * 100
                maxmemory_percent = round(maxmemory_percent, 2)
            data['memory.used_percent'] = float("%.2f" % maxmemory_percent)

        # Iterate over the top level keys
        for key in self._KEYS:
            if self._KEYS[key] in info:
                data[key] = info[self._KEYS[key]]

        # Iterate over renamed keys for 2.6 support
        for key in self._RENAMED_KEYS:
            if self._RENAMED_KEYS[key] in info:
                data[key] = info[self._RENAMED_KEYS[key]]

        # Look for databaase speific stats
        for dbnum in range(0, int(self.config.get('databases',
                                                  self._DATABASE_COUNT))):
            db = 'db%i' % dbnum
            if db in info:
                for key in info[db]:
                    data['%s.%s' % (db, key)] = info[db][key]

        # Time since last save
        for key in ['last_save_time', 'rdb_last_save_time']:
            if key in info:
                data['last_save.time_since'] = int(time.time()) - info[key]

        # Publish the data to graphite
        for key in data:
            self.publish(self._publish_key(nick, key),
                         data[key],
                         precision=self._precision(data[key]),
                         metric_type='GAUGE')
825420
def get_bulkrates(self):
        """"""
        try:
            resp = get(self.bulk_url)
            resp.raise_for_status()
        except exceptions.RequestException as e:
            raise RuntimeError(e)
        return resp.json()
31146
def collect(self):
        
        if not os.access(self.PROC, os.R_OK):
            return False

        # Open PROC file
        file = open(self.PROC, 'r')
        # Get data
        cpuCount = None
        for line in file:
            if not cpuCount:
                cpuCount = len(line.split())
            else:
                data = line.strip().split(None, cpuCount + 2)
                data[0] = data[0].replace(':', '')

                if len(data) == 2:
                    metric_name = data[0]
                    metric_value = data[1]
                    self.publish(metric_name,
                                 self.derivative(metric_name,
                                                 long(metric_value),
                                                 counter))
                else:
                    if len(data[0]) == cpuCount + 1:
                        metric_name = data[0] + '.'
                    elif len(data[0]) == 3:
                        metric_name = (
                            ((data[-2] + ' ' +
                              data[-1]).replace(' ', '_')) + '.')
                    else:
                        metric_name = (
                            ((data[-2]).replace(' ', '_')) +
                            '.' +
                            ((data[-1]).replace(', ', '-').replace(' ', '_')) +
                            '.' + data[0] + '.')
                    total = 0
                    for index, value in enumerate(data):
                        if index == 0 or index >= cpuCount + 1:
                            continue

                        metric_name_node = metric_name + 'CPU' + str(index - 1)
                        value = int(self.derivative(metric_name_node,
                                                    long(value), counter))
                        total += value
                        self.publish(metric_name_node, value)

                    # Roll up value
                    metric_name_node = metric_name + 'total'
                    self.publish(metric_name_node, total)

        # Close file
        file.close()
156753
def _get_total_services_not_monitored(self):
        
        return sum(1 for s in self.services if not s.active_checks_enabled and
                   not s.passive_checks_enabled)
815098
def replicaStatus(self, url):
        """"""
        params = {"f" : "json"}
        url = url + "/status"
        return self._get(url=url,
                            param_dict=params,
                            securityHandler=self._securityHandler,
                            proxy_port=self._proxy_port,
                            proxy_url=self._proxy_url)