Get metrics for each Spark RDD.
481306
def _spark_rdd_metrics(self, instance, running_apps, addl_tags, requests_config):
        
        for app_id, (app_name, tracking_url) in iteritems(running_apps):

            base_url = self._get_request_url(instance, tracking_url)
            response = self._rest_request_to_json(
                base_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, requests_config, addl_tags, app_id, 'storage/rdd'
            )

            tags = ['app_name:%s' % str(app_name)]
            tags.extend(addl_tags)

            for rdd in response:
                self._set_metrics_from_json(tags, rdd, SPARK_RDD_METRICS)

            if len(response):
                self._set_metric('spark.rdd.count', COUNT, len(response), tags)
1086759
def show_partitioning(rdd, show=True):
    """"""
    if show:
        partitionCount = rdd.getNumPartitions()
        try:
            valueCount = rdd.countApprox(1000, confidence=0.50)
        except:
            valueCount = -1
        try:
            name = rdd.name() or None
        except:
            pass
        name = name or "anonymous"
        logging.info("For RDD %s, there are %d partitions with on average %s values" % 
                     (name, partitionCount, int(valueCount/float(partitionCount))))
1049992
def collect(self):
        

        metrics = {}

        if 'scan' in self.config:
            for ld in os.listdir(self.config['owfs']):
                if '.' in ld:
                    self.read_values(ld, self.config['scan'], metrics)

        for oid, files in self.config.iteritems():
            if oid[:3] == 'id:':
                self.read_values(oid[3:], files, metrics)

        for fn, fv in metrics.iteritems():
            self.publish(fn, fv, 2)
946402
def collect_metrics(self, instance):
        
        i_key = self._instance_key(instance)
        if not self.mor_cache.contains(i_key):
            self.log.debug("Not collecting metrics for instance '{}', nothing to do yet.".format(i_key))
            return

        vm_count = 0
        custom_tags = instance.get('tags', [])
        tags = ["vcenter_server:{}".format(ensure_unicode(instance.get('name')))] + custom_tags

        n_mors = self.mor_cache.instance_size(i_key)
        if not n_mors:
            self.gauge('vsphere.vm.count', vm_count, tags=tags)
            self.log.debug("No Mor objects to process for instance '{}', skip...".format(i_key))
            return

        self.log.debug("Collecting metrics for {} mors".format(ensure_unicode(n_mors)))

        # Request metrics for several objects at once. We can limit the number of objects with batch_size
        # If batch_size is 0, process everything at once
        batch_size = self.batch_morlist_size or n_mors
        for batch in self.mor_cache.mors_batch(i_key, batch_size):
            query_specs = []
            for _, mor in iteritems(batch):
                if mor['mor_type'] == 'vm':
                    vm_count += 1
                if mor['mor_type'] not in REALTIME_RESOURCES and ('metrics' not in mor or not mor['metrics']):
                    continue

                query_spec = vim.PerformanceManager.QuerySpec()
                query_spec.entity = mor["mor"]
                query_spec.intervalId = mor["interval"]
                query_spec.maxSample = 1
                if mor['mor_type'] in REALTIME_RESOURCES:
                    query_spec.metricId = self.metadata_cache.get_metric_ids(i_key)
                else:
                    query_spec.metricId = mor["metrics"]
                query_specs.append(query_spec)

            if query_specs:
                self.pool.apply_async(self._collect_metrics_async, args=(instance, query_specs))

        self.gauge('vsphere.vm.count', vm_count, tags=tags)
1152073
def CreateCounterMetadata(metric_name, fields=None, docstring=None, units=None):
  """"""
  return rdf_stats.MetricMetadata(
      varname=metric_name,
      metric_type=rdf_stats.MetricMetadata.MetricType.COUNTER,
      value_type=rdf_stats.MetricMetadata.ValueType.INT,
      fields_defs=FieldDefinitionProtosFromTuples(fields or []),
      docstring=docstring,
      units=units)
943280
def _publish_metrics(self, prev_keys, key, data, publishfn=None):
        """"""
        if key not in data:
            return
        value = data[key]
        keys = prev_keys + [key]
        if not publishfn:
            publishfn = self.publish
        if isinstance(value, dict):
            for new_key in value:
                self._publish_metrics(keys, new_key, value)
        elif isinstance(value, int) or isinstance(value, float):
            publishfn('.'.join(keys), value)
        elif isinstance(value, long):
            publishfn('.'.join(keys), float(value))
563051
public java.util.List<MetricDataResult> getMetricDataResults() {
        if (metricDataResults == null) {
            metricDataResults = new com.amazonaws.internal.SdkInternalList<MetricDataResult>();
        }
        return metricDataResults;
    }
900338
def gather_metrics(self, runs):
        
        for run_dirs in runs.values():
            with open(JSON_METRICS_FILE, 'w') as ostr:
                ostr.write('[\n')
                for i in range(len(run_dirs)):
                    with open(osp.join(run_dirs[i], YAML_REPORT_FILE)) as istr:
                        data = yaml.safe_load(istr)
                        data.pop('category', None)
                        data.pop('command', None)
                        data['id'] = run_dirs[i]
                        json.dump(data, ostr, indent=2)
                    if i != len(run_dirs) - 1:
                        ostr.write(',')
                    ostr.write('\n')
                ostr.write(']\n')
1316394
def _parse_blkio_metrics(self, stats):
        """"""
        metrics = {
            'io_read': 0,
            'io_write': 0,
        }
        for line in stats:
            if 'Read' in line:
                metrics['io_read'] += int(line.split()[2])
            if 'Write' in line:
                metrics['io_write'] += int(line.split()[2])
        return metrics
651801
def get_collection_measures(self):
        """"""

        if not self.quiet:
            print
            print "Computing duration-independent", self.current_collection_type, "measures..."

        self.compute_collection_measures() #include length=1 clusters
        self.compute_collection_measures(no_singletons = True)  #no length=1 clusters

        self.compute_pairwise_similarity_score()

        if not self.quiet:
            collection_measures = [x for x in self.measures \
                               if x.startswith("COLLECTION_")
                                    and self.current_collection_type in x
                                    and self.current_similarity_measure in x]
            collection_measures.sort()
            print_table([(k, str(self.measures[k])) for k in collection_measures])

        if not self.quiet:
            print
            print "Computing duration-based clustering measures..."

        self.compute_duration_measures()
423397
@Override
    protected void collectMetrics(final Metrics metrics) {
        for (final Map.Entry<String, ExecutorService> entry : _executorServices.entrySet()) {
            final String name = entry.getKey();
            final ExecutorService executorService = entry.getValue();
            if (executorService instanceof ForkJoinPool) {
                processForkJoinPool(metrics, name, (ForkJoinPool) executorService);
            }
            if (executorService instanceof ThreadPoolExecutor) {
                // NOTE: That a ScheduledThreadPoolExecutor is a ThreadPoolExecutor
                processThreadPoolExecutor(metrics, name, (ThreadPoolExecutor) executorService);
            }
        }
    }
1302504
private static String[] extract(List<? extends IGenericMetrics> metrics, Extractor e) {
        List<String> strings = new ArrayList<>(metrics.size());
        for (IGenericMetrics metric : metrics) {
            strings.add(e.extract(metric));
        }
        return strings.toArray(new String[strings.size()]);
    }
982134
def collect_metrics(local_evaluator=None,
                    remote_evaluators=[],
                    timeout_seconds=180):
    """"""

    episodes, num_dropped = collect_episodes(
        local_evaluator, remote_evaluators, timeout_seconds=timeout_seconds)
    metrics = summarize_episodes(episodes, episodes, num_dropped)
    return metrics
893198
def available_metrics(self):
        
        req = self.request(self.mist_client.uri+"/clouds/"+self.cloud.id+"/machines/"+self.id+"/metrics")
        metrics = req.get().json()
        return metrics
786157
def _get_spark_app_ids(self, running_apps, requests_config, tags):
        
        spark_apps = {}
        for app_id, (app_name, tracking_url) in iteritems(running_apps):
            response = self._rest_request_to_json(
                tracking_url, SPARK_APPS_PATH, SPARK_SERVICE_CHECK, requests_config, tags
            )

            for app in response:
                app_id = app.get('id')
                app_name = app.get('name')

                if app_id and app_name:
                    spark_apps[app_id] = (app_name, tracking_url)

        return spark_apps
540856
def list_metrics(self, entity, check, limit=None, marker=None,
            return_next=False):
        
        return entity.list_metrics(check, limit=limit, marker=marker,
                return_next=return_next)
1268874
def _collect_stats(self, db, instance_tags):
        

        metric_scope = [self.STATS_METRICS, self.POOLS_METRICS, self.DATABASES_METRICS]

        try:
            with db.cursor(cursor_factory=pgextras.DictCursor) as cursor:
                for scope in metric_scope:
                    descriptors = scope['descriptors']
                    metrics = scope['metrics']
                    query = scope['query']

                    try:
                        self.log.debug("Running query: %s", query)
                        cursor.execute(query)

                        rows = cursor.fetchall()

                    except pg.Error:
                        self.log.exception("Not all metrics may be available")

                    else:
                        for row in rows:
                            self.log.debug("Processing row: %r", row)

                            # Skip the "pgbouncer" database
                            if row['database'] == self.DB_NAME:
                                continue

                            tags = list(instance_tags)
                            tags += ["%s:%s" % (tag, row[column]) for (column, tag) in descriptors if column in row]
                            for (column, (name, reporter)) in metrics:
                                if column in row:
                                    reporter(self, name, row[column], tags)

                        if not rows:
                            self.log.warning("No results were found for query: %s", query)

        except pg.Error:
            self.log.exception("Connection error")

            raise ShouldRestartException