Collect metrics from the monitor backend
199662
def _collect_monitor_metrics(self, conn, tags):
        
        for entry in conn.entries:
            # Get metrics from monitor backend
            dn = entry.entry_dn.lower()
            if dn.endswith(self.CONNECTIONS_METRICS_DN):
                self._handle_connections_entry(entry, tags)
            elif dn.endswith(self.OPERATIONS_METRICS_DN):
                self._handle_operations_entry(entry, tags)
            elif dn.endswith(self.STATISTICS_METRICS_DN):
                self._handle_statistics_entry(entry, tags)
            elif dn.endswith(self.THREADS_METRICS_DN):
                self._handle_threads_entry(entry, tags)
            elif dn.endswith(self.TIME_METRICS_DN):
                self._handle_time_entry(entry, tags)
            elif dn.endswith(self.WAITERS_METRICS_DN):
                self._handle_waiters_entry(entry, tags)
205981
def collect(self):
        
        stats = self.parse_stats_file(self.config["status_path"])
        if len(stats) == 0:
            return {}
        elif "info" not in stats.keys():
            return {}
        elif "programstatus" not in stats.keys():
            return {}

        metrics = self.get_icinga_stats(stats["programstatus"])
        if "hoststatus" in stats.keys():
            metrics = dict(
                metrics.items() + self.get_host_stats(
                    stats["hoststatus"]).items())

        if "servicestatus" in stats.keys():
            metrics = dict(
                metrics.items() + self.get_svc_stats(
                    stats["servicestatus"]).items())

        for metric in metrics.keys():
            self.log.debug("Publishing '%s %s'.", metric, metrics[metric])
            self.publish(metric, metrics[metric])
606661
def getMonitors(self, response_times=0, logs=0, uptime_ratio=''):
        
        url = self.baseUrl
        url += "getMonitors?apiKey=%s" % (self.apiKey)
        url += "&noJsonCallback=1&format=json"
        # responseTimes - optional (defines if the response time data of each
        # monitor will be returned. Should be set to 1 for getting them. Default
        # is 0)
        if response_times:
            url += "&responseTimes=1"
        # logs - optional (defines if the logs of each monitor will be returned.
        # Should be set to 1 for getting the logs. Default is 0)
        if logs:
            url += '&logs=1'
        # customUptimeRatio - optional (defines the number of days to calculate
        # the uptime ratio(s) for. Ex: customUptimeRatio=7-30-45 to get the
        # uptime ratios for those periods)
        if uptime_ratio:
            url += '&customUptimeRatio=%s' % uptime_ratio

        return self.requestApi(url)
1336978
def monitor(self, operation='', **kw):
        
        mon = self._monitor(operation, hdf5=self.datastore.hdf5)
        self._monitor.calc_id = mon.calc_id = self.datastore.calc_id
        vars(mon).update(kw)
        return mon
571760
def collect_metrics():
    
    def _register(action):
        handler = Handler.get(action)
        handler.add_predicate(partial(_restricted_hook, 'collect-metrics'))
        return action
    return _register
630436
def metrics(self):
        
        masterThrp, backupThrp = self.getThroughputs(self.instances.masterId)
        r = self.instance_throughput_ratio(self.instances.masterId)
        m = [
            ("{} Monitor metrics:".format(self), None),
            ("Delta", self.Delta),
            ("Lambda", self.Lambda),
            ("Omega", self.Omega),
            ("instances started", self.instances.started),
            ("ordered request counts",
             {i: r[0] for i, r in self.numOrderedRequests.items()}),
            ("ordered request durations",
             {i: r[1] for i, r in self.numOrderedRequests.items()}),
            ("master request latencies", self.masterReqLatencies),
            ("client avg request latencies", {i: self.getLatency(i)
                                              for i in self.instances.ids}),
            ("throughput", {i: self.getThroughput(i)
                            for i in self.instances.ids}),
            ("master throughput", masterThrp),
            ("total requests", self.totalRequests),
            ("avg backup throughput", backupThrp),
            ("master throughput ratio", r)]
        return m
1012602
def getMonitors(self, apikey, monitors='none'):
        """"""
        data= 'api_key=' + apikey + '&format=json'
        if monitors != 'none':
            data += '&monitors=' + monitors
        fetchUrl = self.BASE_URL
        fetchUrl += 'getMonitors'
        try:
            monitors = requests.post(fetchUrl,
                headers=self.headers, data=data, timeout=3)
        except:
            return False
        else:
            if monitors:
                return monitors.json()
            else:
                return False
297101
def collect(self):
        
        if not psutil:
            self.log.error('Unable to import psutil')
            self.log.error('No process resource metrics retrieved')
            return None

        for process in psutil.process_iter():
            self.collect_process_info(process)

        # publish results
        for pg_name, counters in self.processes_info.iteritems():
            if counters:
                metrics = (
                    ("%s.%s" % (pg_name, key), value)
                    for key, value in counters.iteritems())
            else:
                if self.processes[pg_name]['count_workers']:
                    metrics = (('%s.workers_count' % pg_name, 0), )
                else:
                    metrics = ()

            [self.publish(*metric) for metric in metrics]
            # reinitialize process info
            self.processes_info[pg_name] = {}
549115
def addMonitor(self, monitor):
    

    token = self.nextMonitorToken
    self.nextMonitorToken += 1

    self.monitors[token] = monitor

    return token
874357
def get_metrics(awsclient, name):
    
    metrics = ['Duration', 'Errors', 'Invocations', 'Throttles']
    client_cw = awsclient.get_client('cloudwatch')
    for metric in metrics:
        response = client_cw.get_metric_statistics(
            Namespace='AWS/Lambda',
            MetricName=metric,
            Dimensions=[
                {
                    'Name': 'FunctionName',
                    'Value': name
                },
            ],
            # StartTime=datetime.now() + timedelta(days=-1),
            # EndTime=datetime.now(),
            StartTime=maya.now().subtract(days=1).datetime(),
            EndTime=maya.now().datetime(),
            Period=3600,
            Statistics=[
                'Sum',
            ],
            Unit=unit(metric)
        )
        log.info('\t%s %s' % (metric,
                           repr(aggregate_datapoints(response['Datapoints']))))
    return 0
696446
def _report_self(self):
        '''
        
        '''
        key = "stats:kafka-monitor:self:{m}:{u}".format(
            m=socket.gethostname(),
            u=self.my_uuid)
        self.redis_conn.set(key, time.time())
        self.redis_conn.expire(key, self.settings['HEARTBEAT_TIMEOUT'])
954843
private void collectPoolInfoMetrics() {
    Map<PoolInfo, PoolInfoMetrics> newPoolNameToMetrics = new HashMap<PoolInfo,
      PoolInfoMetrics>();
    long now = ClusterManager.clock.getTime();

    Map<PoolInfo, Long> poolInfoAverageFirstWaitMs =
        sessionManager.getTypePoolInfoAveFirstWaitMs(type);

    // The gets + puts below are OK because only one thread is doing it.
    for (PoolGroupSchedulable poolGroup : poolGroupManager.getPoolGroups()) {
      int poolGroupSessions = 0;
      for (PoolSchedulable pool : poolGroup.getPools()) {
        MetricsRecord poolRecord =
            poolInfoToMetricsRecord.get(pool.getPoolInfo());
        if (poolRecord == null) {
          poolRecord = metrics.getContext().createRecord(
              "pool-" + pool.getName());
          poolInfoToMetricsRecord.put(pool.getPoolInfo(), poolRecord);
        }

        PoolInfoMetrics poolMetrics = new PoolInfoMetrics(pool.getPoolInfo(),
            type, poolRecord);
        poolMetrics.setCounter(
            MetricName.GRANTED, pool.getGranted());
        poolMetrics.setCounter(
            MetricName.REQUESTED, pool.getRequested());
        poolMetrics.setCounter(
            MetricName.SHARE, (long) pool.getShare());
        poolMetrics.setCounter(
            MetricName.MIN, pool.getMinimum());
        poolMetrics.setCounter(
            MetricName.MAX, pool.getMaximum());
        poolMetrics.setCounter(
            MetricName.WEIGHT, (long) pool.getWeight());
        poolMetrics.setCounter(
            MetricName.SESSIONS, pool.getScheduleQueue().size());
        poolMetrics.setCounter(
            MetricName.STARVING, pool.getStarvingTime(now) / 1000);
        Long averageFirstTypeMs =
            poolInfoAverageFirstWaitMs.get(pool.getPoolInfo());
        poolMetrics.setCounter(MetricName.AVE_FIRST_WAIT_MS,
            (averageFirstTypeMs == null) ?
                0 : averageFirstTypeMs.longValue());

        newPoolNameToMetrics.put(pool.getPoolInfo(), poolMetrics);
        poolGroupSessions += pool.getScheduleQueue().size();
      }

      MetricsRecord poolGroupRecord =
          poolInfoToMetricsRecord.get(poolGroup.getName());
      if (poolGroupRecord == null) {
        poolGroupRecord = metrics.getContext().createRecord(
            "poolgroup-" + poolGroup.getName());
        poolInfoToMetricsRecord.put(poolGroup.getPoolInfo(), poolGroupRecord);
      }

      PoolInfoMetrics poolGroupMetrics =
          new PoolInfoMetrics(poolGroup.getPoolInfo(), type, poolGroupRecord);
      poolGroupMetrics.setCounter(
          MetricName.GRANTED, poolGroup.getGranted());
      poolGroupMetrics.setCounter(
          MetricName.REQUESTED, poolGroup.getRequested());
      poolGroupMetrics.setCounter(
          MetricName.SHARE, (long) poolGroup.getShare());
      poolGroupMetrics.setCounter(
          MetricName.MIN, poolGroup.getMinimum());
      poolGroupMetrics.setCounter(
          MetricName.MAX, poolGroup.getMaximum());
      poolGroupMetrics.setCounter(
          MetricName.SESSIONS, poolGroupSessions);
      newPoolNameToMetrics.put(poolGroup.getPoolInfo(), poolGroupMetrics);
    }

    poolInfoToMetrics = newPoolNameToMetrics;
  }
403031
def statsd_metric(name, count, elapsed):
    """"""
    with statsd.pipeline() as pipe:
        pipe.incr(name, count)
        pipe.timing(name, int(round(1000 * elapsed)))
339430
def scrape_metrics(self, endpoint):
        
        response = self.poll(endpoint)
        try:
            # no dry run if no label joins
            if not self.label_joins:
                self._dry_run = False
            elif not self._watched_labels:
                # build the _watched_labels set
                for val in itervalues(self.label_joins):
                    self._watched_labels.add(val['label_to_match'])

            for metric in self.parse_metric_family(response):
                yield metric

            # Set dry run off
            self._dry_run = False
            # Garbage collect unused mapping and reset active labels
            for metric, mapping in list(iteritems(self._label_mapping)):
                for key in list(mapping):
                    if key not in self._active_label_mapping[metric]:
                        del self._label_mapping[metric][key]
            self._active_label_mapping = {}
        finally:
            response.close()
446953
def monitor_counters(mc, output, counters, detailed, f):
    """"""
    # Print CSV header
    output.write("time,{}{}\n".format("x,y," if detailed else "",
                                      ",".join(counters)))

    system_info = mc.get_system_info()

    # Make an initial sample of the counters
    last_counter_values = sample_counters(mc, system_info)

    start_time = time.time()

    for _ in f():
        # Snapshot the change in counter values
        counter_values = sample_counters(mc, system_info)
        delta = deltas(last_counter_values, counter_values)
        last_counter_values = counter_values

        now = time.time() - start_time

        # Output the changes
        if detailed:
            for x, y in sorted(system_info):
                output.write("{:0.1f},{},{},{}\n".format(
                    now, x, y,
                    ",".join(str(getattr(delta[(x, y)], c))
                             for c in counters)))
        else:
            totals = [0 for _ in counters]
            for xy in sorted(system_info):
                for i, counter in enumerate(counters):
                    totals[i] += getattr(delta[xy], counter)
            output.write("{:0.1f},{}\n".format(
                now, ",".join(map(str, totals))))
470335
def _get_metric(self, metric, tablename, index_name=None):
        
        end = time.time()
        begin = end - 3 * 60  # 3 minute window
        dimensions = [{"Name": "TableName", "Value": tablename}]
        if index_name is not None:
            dimensions.append({"Name": "GlobalSecondaryIndexName", "Value": index_name})
        period = 60
        data = self.cloudwatch_connection.get_metric_statistics(
            Period=period,
            StartTime=begin,
            EndTime=end,
            MetricName=metric,
            Namespace="AWS/DynamoDB",
            Statistics=["Sum"],
            Dimensions=dimensions,
        )
        points = data["Datapoints"]
        if not points:
            return 0
        else:
            points.sort(key=lambda r: r["Timestamp"])
            return float(points[-1]["Sum"]) / period
693921
def collect(self):
        

        def cpu_time_list():
            """
            get cpu time list
            """

            statFile = open(self.PROC, "r")
            timeList = statFile.readline().split(" ")[2:6]
            for i in range(len(timeList)):
                timeList[i] = int(timeList[i])
            statFile.close()
            return timeList

        def cpu_delta_time(interval):
            """
            Get before and after cpu times for usage calc
            """
            pre_check = cpu_time_list()
            time.sleep(interval)
            post_check = cpu_time_list()
            for i in range(len(pre_check)):
                post_check[i] -= pre_check[i]
            return post_check

        if os.access(self.PROC, os.R_OK):

            # If simple only return aggregate CPU% metric
            if str_to_bool(self.config['simple']):
                dt = cpu_delta_time(self.INTERVAL)
                cpuPct = 100 - (dt[len(dt) - 1] * 100.00 / sum(dt))
                self.publish('percent', str('%.4f' % cpuPct))
                return True

            results = {}
            # Open file
            file = open(self.PROC)

            ncpus = -1  # dont want to count the 'cpu'(total) cpu.
            for line in file:
                if not line.startswith('cpu'):
                    continue

                ncpus += 1
                elements = line.split()

                cpu = elements[0]

                if cpu == 'cpu':
                    cpu = 'total'
                elif not str_to_bool(self.config['percore']):
                    continue

                results[cpu] = {}

                if len(elements) >= 2:
                    results[cpu]['user'] = elements[1]
                if len(elements) >= 3:
                    results[cpu]['nice'] = elements[2]
                if len(elements) >= 4:
                    results[cpu]['system'] = elements[3]
                if len(elements) >= 5:
                    results[cpu]['idle'] = elements[4]
                if len(elements) >= 6:
                    results[cpu]['iowait'] = elements[5]
                if len(elements) >= 7:
                    results[cpu]['irq'] = elements[6]
                if len(elements) >= 8:
                    results[cpu]['softirq'] = elements[7]
                if len(elements) >= 9:
                    results[cpu]['steal'] = elements[8]
                if len(elements) >= 10:
                    results[cpu]['guest'] = elements[9]
                if len(elements) >= 11:
                    results[cpu]['guest_nice'] = elements[10]

            # Close File
            file.close()

            metrics = {'cpu_count': ncpus}

            for cpu in results.keys():
                stats = results[cpu]
                for s in stats.keys():
                    # Get Metric Name
                    metric_name = '.'.join([cpu, s])
                    # Get actual data
                    if ((str_to_bool(self.config['normalize']) and
                         cpu == 'total' and
                         ncpus > 0)):
                        metrics[metric_name] = self.derivative(
                            metric_name,
                            long(stats[s]),
                            self.MAX_VALUES[s]) / ncpus
                    else:
                        metrics[metric_name] = self.derivative(
                            metric_name,
                            long(stats[s]),
                            self.MAX_VALUES[s])

            # Check for a bug in xen where the idle time is doubled for guest
            # See https://bugzilla.redhat.com/show_bug.cgi?id=624756
            if self.config['xenfix'] is None or self.config['xenfix'] is True:
                if os.path.isdir('/proc/xen'):
                    total = 0
                    for metric_name in metrics.keys():
                        if 'cpu0.' in metric_name:
                            total += int(metrics[metric_name])
                    if total > 110:
                        self.config['xenfix'] = True
                        for mname in metrics.keys():
                            if '.idle' in mname:
                                metrics[mname] = float(metrics[mname]) / 2
                    elif total > 0:
                        self.config['xenfix'] = False
                else:
                    self.config['xenfix'] = False

            # Publish Metric Derivative
            for metric_name in metrics.keys():
                self.publish(metric_name,
                             metrics[metric_name],
                             precision=2)
            return True

        else:
            if not psutil:
                self.log.error('Unable to import psutil')
                self.log.error('No cpu metrics retrieved')
                return None

            cpu_time = psutil.cpu_times(True)
            cpu_count = len(cpu_time)
            total_time = psutil.cpu_times()
            for i in range(0, len(cpu_time)):
                metric_name = 'cpu' + str(i)
                self.publish(
                    metric_name + '.user',
                    self.derivative(metric_name + '.user',
                                    cpu_time[i].user,
                                    self.MAX_VALUES['user']),
                    precision=2)

                if hasattr(cpu_time[i], 'nice'):
                    self.publish(
                        metric_name + '.nice',
                        self.derivative(metric_name + '.nice',
                                        cpu_time[i].nice,
                                        self.MAX_VALUES['nice']),
                        precision=2)

                self.publish(
                    metric_name + '.system',
                    self.derivative(metric_name + '.system',
                                    cpu_time[i].system,
                                    self.MAX_VALUES['system']),
                    precision=2)

                self.publish(
                    metric_name + '.idle',
                    self.derivative(metric_name + '.idle',
                                    cpu_time[i].idle,
                                    self.MAX_VALUES['idle']),
                    precision=2)

            metric_name = 'total'
            self.publish(
                metric_name + '.user',
                self.derivative(metric_name + '.user',
                                total_time.user,
                                self.MAX_VALUES['user']) / cpu_count,
                precision=2)

            if hasattr(total_time, 'nice'):
                self.publish(
                    metric_name + '.nice',
                    self.derivative(metric_name + '.nice',
                                    total_time.nice,
                                    self.MAX_VALUES['nice']) / cpu_count,
                    precision=2)

            self.publish(
                metric_name + '.system',
                self.derivative(metric_name + '.system',
                                total_time.system,
                                self.MAX_VALUES['system']) / cpu_count,
                precision=2)

            self.publish(
                metric_name + '.idle',
                self.derivative(metric_name + '.idle',
                                total_time.idle,
                                self.MAX_VALUES['idle']) / cpu_count,
                precision=2)

            self.publish('cpu_count', psutil.cpu_count())

            return True

        return None