Interactively parse metric info from a Prometheus endpoint.
327649
def parse(ctx, endpoint, check, here):
    """"""
    if here:
        output_dir = os.getcwd()
    else:
        output_dir = path_join(get_root(), check)
        if not dir_exists(output_dir):
            abort(
                'Check `{check}` does not exist; try `ddev create{repo_flag} {check}`.'.format(
                    check=check, repo_flag=' -e' if ctx.obj['repo_choice'] == 'extras' else ''
                )
            )

    endpoint = sanitize_endpoint(endpoint)

    echo_waiting('Scraping `{}`...'.format(endpoint))
    metrics = parse_metrics(endpoint)
    num_metrics = len(metrics)

    echo_success('\nGlobally available options:')
    echo_info('    t - Append .total to the available options')
    echo_info('    s - Skip')
    echo_info('    q - Quit')

    for i, (metric, data) in enumerate(sorted(iteritems(metrics)), 1):
        metric_parts = metric.split('_')
        metric_template = '{}'.join(metric_parts)
        num_separators = len(metric_parts) - 1

        metric_options = [
            metric_template.format(*possible_separators)
            for possible_separators in product(METRIC_SEPARATORS, repeat=num_separators)
        ]
        num_options = len(metric_options)

        default_option = num_options
        options_prompt = 'Choose an option (default {}, as-is): '.format(default_option)
        options_text = get_options_text(metric_options)

        finished = False
        choice_error = ''
        progress_status = '({} of {}) '.format(i, num_metrics)
        indent = ' ' * len(progress_status)

        while not finished:
            echo_success('\n{}{}'.format(progress_status, metric))

            echo_success('Type: ', nl=False, indent=indent)
            echo_info(data.get('type', 'None'))

            echo_success('Info: ', nl=False, indent=indent)
            echo_info(data.get('description', 'None'))

            echo_info(options_text)

            if choice_error:
                echo_warning(choice_error)

            echo_waiting(options_prompt, nl=False)

            if num_options >= 9:
                choice = ensure_unicode(input())
            else:
                # Terminals are odd and sometimes produce an erroneous null byte
                choice = '\x00'
                while choice == '\x00':
                    choice = click.getchar().strip()
                    try:
                        choice = ensure_unicode(choice)
                    except UnicodeDecodeError:
                        choice = repr(choice)

            if not choice:
                choice = default_option

            if choice == 't':
                echo_info('Append .total')
                for n in range(num_options):
                    metric_options[n] += '.total'
                options_text = get_options_text(metric_options)
                continue
            elif choice == 's':
                echo_info('Skip')
                echo_info('Skipped {}'.format(metric))
                break
            elif choice == 'q':
                echo_info('Exit')
                echo_warning('Exited at {}'.format(metric))
                return

            try:
                choice = int(choice)
            except Exception:
                pass

            if choice not in range(1, num_options + 1):
                echo_info(u'{}'.format(choice))
                choice_error = u'`{}` is not a valid option.'.format(choice)
                continue
            else:
                choice_error = ''

            option = metric_options[choice - 1]
            echo_info(option)

            data['dd_name'] = option

            finished = True

    metadata_file = path_join(output_dir, 'metadata.csv')
    echo_waiting('\nWriting `{}`... '.format(metadata_file), nl=False)

    metric_items = sorted(iteritems(metrics), key=lambda item: item[1]['dd_name'])
    output_lines = ['{}\n'.format(METADATA_CSV_HEADER)]
    for _, data in metric_items:
        metric_name = data['dd_name']
        metric_type = TYPE_MAP.get(data.get('type'), '')
        metric_description = data.get('description', '')
        if ',' in metric_description:
            metric_description = '"{}"'.format(metric_description)

        output_lines.append(
            '{check}.{metric_name},{metric_type},,,,{metric_description},0,{check},\n'.format(
                check=check, metric_name=metric_name, metric_type=metric_type, metric_description=metric_description
            )
        )

    write_file_lines(metadata_file, output_lines)
    echo_success('success!')

    metric_map = (
        'METRIC_MAP = {{\n'
        '{}\n'
        '}}'.format('\n'.join("    '{}': '{}',".format(metric, data['dd_name']) for metric, data in metric_items))
    )

    pyperclip.copy(metric_map)
    echo_success('\nThe metric map has been copied to your clipboard, paste it to any file you want!')
1210309
def log_metrics(self, metrics_by_name, info):
        
        try:
            metrics_path = os.path.join(self.dir, "metrics.json")
            with open(metrics_path, 'r') as f:
                saved_metrics = json.load(f)
        except IOError:
            # We haven't recorded anything yet. Start Collecting.
            saved_metrics = {}

        for metric_name, metric_ptr in metrics_by_name.items():

            if metric_name not in saved_metrics:
                saved_metrics[metric_name] = {"values": [],
                                              "steps": [],
                                              "timestamps": []}

            saved_metrics[metric_name]["values"] += metric_ptr["values"]
            saved_metrics[metric_name]["steps"] += metric_ptr["steps"]

            # Manually convert them to avoid passing a datetime dtype handler
            # when we're trying to convert into json.
            timestamps_norm = [ts.isoformat()
                               for ts in metric_ptr["timestamps"]]
            saved_metrics[metric_name]["timestamps"] += timestamps_norm

        self.save_json(saved_metrics, 'metrics.json')
666418
def get_metrics_from_perfdata(self, service, perf_data):
        """"""
        result = []
        metrics = PerfDatas(perf_data)

        for metric in metrics:
            logger.debug("service: %s, metric: %s (%s)", service, metric, metric.__dict__)

            if metric.name in ['time']:
                metric.name = "duration"
            name = sanitize_name(metric.name)
            name = self.multiple_values.sub(r'.\1', name)
            if not name:
                continue

            # get metric value and its thresholds values if they exist
            name_value = {
                name: metric.value,
                'uom_' + name: metric.uom
            }

            # Get or ignore extra values depending upon module configuration
            if metric.warning and self.send_warning:
                name_value[name + '_warn'] = metric.warning

            if metric.critical and self.send_critical:
                name_value[name + '_crit'] = metric.critical

            if metric.min and self.send_min:
                name_value[name + '_min'] = metric.min

            if metric.max and self.send_max:
                name_value[name + '_max'] = metric.max

            for key, value in name_value.items():
                result.append((key, value, metric.uom))

        logger.debug("Metrics: %s - %s", service, result)
        return result
1058271
def process_metrics(self, snmp_data):
        ""
        self.metrics = {}
        for i in range(0, len(snmp_data)):
            mib_name = self.models[self.modem_type]['metrics'][i]
            conv = metric_mib[mib_name]['conv']
            self.metrics[mib_name] = conv(snmp_data[i])
581830
public static Stream<PrometheusMetric> filteredMetrics(PullProcessorPipeline registry) throws Exception {
        Stream <PrometheusMetric> m = registry.get().stream()
                .flatMap((TimeSeriesValue i) -> {
                    Map<MetricName, MetricValue> metrics = i.getMetrics();
                    GroupName group = i.getGroup();
                    return metrics.entrySet().stream()
                            .filter((kv) -> kv.getValue().value().isPresent())
                            .map((kv) -> {
                                final String metric_group = toPrometheusString_(group.getPath().getPath());
                                final Map<String, String> tags = group.getTags().stream()
                                        .filter(tag -> tag.getValue().asString().isPresent())
                                        .collect(Collectors.toMap(
                                                tag -> escapeprometheus(tag.getKey()),
                                                tag -> escapeLabelValue_(tag.getValue().asString().get())));
                                final String metric_name = toPrometheusString_(kv.getKey().getPath());
                                final Number metric_value = kv.getValue().value().get();
                                return new PrometheusMetric(metric_group, tags, metric_name, metric_value);
                            });
                }
792869
def process(self, metric):
        
        self.log.debug("Metric: %s", str(metric).rstrip().replace(' ', '\t'))
1313341
def gather_data(registry):
    """"""

    # Get the host name of the machine
    host = socket.gethostname()

    # Create our collectors
    io_metric = Summary("write_file_io_example",
                        "Writing io file in disk example.",
                        {'host': host})

    # register the metric collectors
    registry.register(io_metric)
    chunk = b'\xff'*4000  # 4000 bytes
    filename_path = "/tmp/prometheus_test"
    blocksizes = (100, 10000, 1000000, 100000000)

    # Start gathering metrics every 0.7 seconds
    while True:
        time.sleep(0.7)

        for i in blocksizes:
            time_start = time.time()
            # Action
            with open(filename_path, "wb") as f:
                for _ in range(i // 10000):
                    f.write(chunk)

            io_metric.add({"file": filename_path, "block": i},
                          time.time() - time_start)
637410
def _collect_for_instance(self, instance, connection):
        """"""
        with connection.cursor() as cursor:
            for queue, metrics in self.get_queue_info(instance, cursor):
                for name, metric in metrics.items():
                    self.publish('.'.join((instance, queue, name)), metric)

        with connection.cursor() as cursor:
            consumers = self.get_consumer_info(instance, cursor)
            for queue, consumer, metrics in consumers:
                for name, metric in metrics.items():
                    key_parts = (instance, queue, 'consumers', consumer, name)
                    self.publish('.'.join(key_parts), metric)
87947
def _flatten_json(cls, metric_base, val, tags):
        
        output = []

        if isinstance(val, dict):
            # Pull out the server as a tag instead of trying to read as a metric
            if 'server' in val and val['server']:
                server = 'server:%s' % val.pop('server')
                if tags is None:
                    tags = [server]
                else:
                    tags = tags + [server]
            for key, val2 in iteritems(val):
                if key in TAGGED_KEYS:
                    metric_name = '%s.%s' % (metric_base, TAGGED_KEYS[key])
                    for tag_val, data in iteritems(val2):
                        tag = '%s:%s' % (TAGGED_KEYS[key], tag_val)
                        output.extend(cls._flatten_json(metric_name, data, tags + [tag]))
                else:
                    metric_name = '%s.%s' % (metric_base, key)
                    output.extend(cls._flatten_json(metric_name, val2, tags))

        elif isinstance(val, list):
            for val2 in val:
                output.extend(cls._flatten_json(metric_base, val2, tags))

        elif isinstance(val, bool):
            output.append((metric_base, int(val), tags, 'gauge'))

        elif isinstance(val, (int, float, long)):
            output.append((metric_base, val, tags, 'gauge'))

        elif isinstance(val, (text_type, str)):
            if val[-1] == "Z":
                try:
                    # In the new Plus API, timestamps are now formatted
                    # strings, some include microseconds, some don't...
                    timestamp = fromisoformat(val[:19])
                except ValueError:
                    pass
                else:
                    output.append((metric_base, int((timestamp - EPOCH).total_seconds()), tags, 'gauge'))

        return output
874207
def _extract_metric_from_map(self, _m, messages, obj_map, obj_help):
        
        _obj = metrics_pb2.MetricFamily()
        _obj.name = _m
        _obj.type = self.METRIC_TYPES.index(obj_map[_m])
        if _m in obj_help:
            _obj.help = obj_help[_m]
        # trick for histograms
        _newlbl = _m
        if obj_map[_m] == 'histogram':
            _newlbl = '{}_bucket'.format(_m)
        # Loop through the array of metrics ({labels, value}) built earlier
        for _metric in messages[_newlbl]:
            # in the case of quantiles and buckets, they need to be grouped by labels
            if obj_map[_m] in ['summary', 'histogram'] and len(_obj.metric) > 0:
                _label_exists = False
                _metric_minus = {k: v for k, v in list(iteritems(_metric['labels'])) if k not in ['quantile', 'le']}
                _metric_idx = 0
                for mls in _obj.metric:
                    _tmp_lbl = {idx.name: idx.value for idx in mls.label}
                    if _metric_minus == _tmp_lbl:
                        _label_exists = True
                        break
                    _metric_idx = _metric_idx + 1
                if _label_exists:
                    _g = _obj.metric[_metric_idx]
                else:
                    _g = _obj.metric.add()
            else:
                _g = _obj.metric.add()
            if obj_map[_m] == 'counter':
                _g.counter.value = float(_metric['value'])
            elif obj_map[_m] == 'gauge':
                _g.gauge.value = float(_metric['value'])
            elif obj_map[_m] == 'summary':
                if '{}_count'.format(_m) in messages:
                    _g.summary.sample_count = long(self.get_metric_value_by_labels(messages, _metric, _m, 'count'))
                if '{}_sum'.format(_m) in messages:
                    _g.summary.sample_sum = self.get_metric_value_by_labels(messages, _metric, _m, 'sum')
            # TODO: see what can be done with the untyped metrics
            elif obj_map[_m] == 'histogram':
                if '{}_count'.format(_m) in messages:
                    _g.histogram.sample_count = long(self.get_metric_value_by_labels(messages, _metric, _m, 'count'))
                if '{}_sum'.format(_m) in messages:
                    _g.histogram.sample_sum = self.get_metric_value_by_labels(messages, _metric, _m, 'sum')
            # last_metric = len(_obj.metric) - 1
            # if last_metric >= 0:
            for lbl in _metric['labels']:
                # In the string format, the quantiles are in the labels
                if lbl == 'quantile':
                    # _q = _obj.metric[last_metric].summary.quantile.add()
                    _q = _g.summary.quantile.add()
                    _q.quantile = float(_metric['labels'][lbl])
                    _q.value = float(_metric['value'])
                # The upper_bounds are stored as "le" labels on string format
                elif obj_map[_m] == 'histogram' and lbl == 'le':
                    # _q = _obj.metric[last_metric].histogram.bucket.add()
                    _q = _g.histogram.bucket.add()
                    _q.upper_bound = float(_metric['labels'][lbl])
                    _q.cumulative_count = long(float(_metric['value']))
                else:
                    # labels deduplication
                    is_in_labels = False
                    for _existing_lbl in _g.label:
                        if lbl == _existing_lbl.name:
                            is_in_labels = True
                    if not is_in_labels:
                        _l = _g.label.add()
                        _l.name = lbl
                        _l.value = _metric['labels'][lbl]
        return _obj
294572
def _process_metric(self, data, metric, xtype, path, xform=None, tags=None, hostname=None):
        
        value = data

        # Traverse the nested dictionaries
        for key in path.split('.'):
            if value is not None:
                value = value.get(key)
            else:
                break

        if value is not None:
            if xform:
                value = xform(value)
            if xtype == "gauge":
                self.gauge(metric, value, tags=tags, hostname=hostname)
            else:
                self.rate(metric, value, tags=tags, hostname=hostname)
        else:
            self.log.debug("Metric not found: %s -> %s", path, metric)
75587
def show_metrics(metrics, all_branches=False, all_tags=False):
    
    for branch, val in metrics.items():
        if all_branches or all_tags:
            logger.info("{branch}:".format(branch=branch))

        for fname, metric in val.items():
            lines = metric if type(metric) is list else metric.splitlines()

            if len(lines) > 1:
                logger.info("\t{fname}:".format(fname=fname))

                for line in lines:
                    logger.info("\t\t{content}".format(content=line))

            else:
                logger.info("\t{}: {}".format(fname, metric))
1267103
async def get_metrics(self, element):
        """"""
        await self.get_data()
        await self.get_plugins()

        if element in self.plugins:
            self.values = self.data[element]
        else:
            raise exceptions.GlancesApiError("Element data not available")
1324543
def metric_details(request, pk, format=None):
    
    metric = get_object_or_404(Metric, pk=pk)
    # get
    if request.method == 'GET':
        try:
            results = metric.select(q=request.query_params.get('q', metric.query))
        except InfluxDBClientError as e:
            return Response({'detail': e.content}, status=e.code)
        return Response(list(results.get_points(metric.name)))
    # post
    else:
        if not request.data:
            return Response({'detail': 'expected values in POST data or JSON payload'},
                            status=400)
        data = request.data.copy()
        # try converting strings to floats when sending form-data
        if request.content_type != 'application/json':
            for key, value in data.items():
                try:
                    data[key] = float(value) if '.' in value else int(value)
                except ValueError:
                    pass
        # write
        metric.write(data)
        return Response({'detail': 'ok'})
215863
private Iterable<PutMetricDataRequest> toPutMetricDataRequests(Map<String, MetricDatum> uniqueMetrics) {
        // Opportunistically generates some machine metrics whenever there
        // is metrics consolidation
        for (MetricDatum datum: machineMetricFactory.generateMetrics()) {
            summarize(datum, uniqueMetrics);
        }
        List<PutMetricDataRequest> list = new ArrayList<PutMetricDataRequest>();
        List<MetricDatum> data = new ArrayList<MetricDatum>();
        for (MetricDatum m: uniqueMetrics.values()) {
            data.add(m);
            if (data.size() == CloudWatchMetricConfig.MAX_METRICS_DATUM_SIZE) {
                list.addAll(newPutMetricDataRequests(data));
                data.clear();
            }
        }

        if (data.size() > 0) {
            list.addAll(newPutMetricDataRequests(data));
        }
        return list;
    }
554161
def SetupPrometheusExportsFromConfig():
    """"""
    port = getattr(settings, 'PROMETHEUS_METRICS_EXPORT_PORT', None)
    port_range = getattr(
        settings, 'PROMETHEUS_METRICS_EXPORT_PORT_RANGE', None)
    addr = getattr(settings, 'PROMETHEUS_METRICS_EXPORT_ADDRESS', '')
    if port_range:
        SetupPrometheusEndpointOnPortRange(port_range, addr)
    elif port:
        SetupPrometheusEndpointOnPort(port, addr)
1108551
def parse(cls, string):
        
        match = re.match(r'^(?P<name>[A-Za-z0-9\.\-_]+)\s+' +
                         '(?P<value>[0-9\.]+)\s+' +
                         '(?P<timestamp>[0-9\.]+)(\n?)$',
                         string)
        try:
            groups = match.groupdict()
            # TODO: get precision from value string
            return Metric(groups['name'],
                          groups['value'],
                          float(groups['timestamp']))
        except:
            raise DiamondException(
                "Metric could not be parsed from string: %s." % string)