Parse and submit the metrics from a file
486820
def _process_file(self, path):
        
        try:
            f = open(path)
            for line in f:
                self._process_line(line)

            os.remove(path)
        except IOError as ex:
            self.log.error("Could not open file `{path}': {error}".format(
                path=path, error=ex.strerror))
158073
def process(self, metric):
        
        entry = {'timestamp': metric.timestamp, 'value': metric.value,
                 "tags": {}}
        entry["tags"]["hostname"] = metric.host

        if self.cleanMetrics:
            metric = MetricWrapper(metric, self.log)
            if self.skipAggregates and metric.isAggregate():
                return
            for tagKey in metric.getTags():
                entry["tags"][tagKey] = metric.getTags()[tagKey]

        entry['metric'] = (self.prefix + metric.getCollectorPath() +
                           '.' + metric.getMetricPath())

        for [key, value] in self.tags:
            entry["tags"][key] = value

        self.entrys.append(entry)

        # send data if list is long enough
        if (len(self.entrys) >= self.batch):
            # Compress data
            if self.compression >= 1:
                data = StringIO.StringIO()
                with contextlib.closing(gzip.GzipFile(fileobj=data,
                                        compresslevel=self.compression,
                                        mode="w")) as f:
                    f.write(json.dumps(self.entrys))
                self._send(data.getvalue())
            else:
                # no compression
                data = json.dumps(self.entrys)
                self._send(data)
751702
def _parse_perfdata(self, s):
        
        metrics = []
        counters = re.findall(self.TOKENIZER_RE, s)
        if counters is None:
            self.log.warning("Failed to parse performance data: {s}".format(
                s=s))
            return metrics

        for (key, value, uom, warn, crit, min, max) in counters:
            try:
                norm_value = self._normalize_to_unit(float(value), uom)
                metrics.append((key, norm_value))
            except ValueError:
                self.log.warning(
                    "Couldn't convert value '{value}' to float".format(
                        value=value))

        return metrics
1208339
def write_metrics_file(metrics: List[Dict[str, Any]], path: str):
    
    with open(path, 'w') as metrics_out:
        for checkpoint, metric_dict in enumerate(metrics, 1):
            metrics_str = "\t".join(["{}={}".format(name, value) for name, value in sorted(metric_dict.items())])
            metrics_out.write("{}\t{}\n".format(checkpoint, metrics_str))
1159220
def get_file_stats(file_name, entity_type='file', lineno=None, cursorpos=None,
                   plugin=None, language=None, local_file=None):
    """"""

    language = standardize_language(language, plugin)
    stats = {
        'language': language,
        'dependencies': [],
        'lines': None,
        'lineno': lineno,
        'cursorpos': cursorpos,
    }

    if entity_type == 'file':
        lexer = get_lexer(language)
        if not language:
            language, lexer = guess_language(file_name, local_file)
        parser = DependencyParser(local_file or file_name, lexer)
        stats.update({
            'language': use_root_language(language, lexer),
            'dependencies': parser.parse(),
            'lines': number_lines_in_file(local_file or file_name),
        })

    return stats
424486
def format(file_metrics, build_metrics):
    """"""
    metrics = {'files': file_metrics}
    if build_metrics:
        metrics['build'] = build_metrics
    body = json.dumps(metrics, sort_keys=True, indent=4) + '\n'
    return body
900338
def gather_metrics(self, runs):
        
        for run_dirs in runs.values():
            with open(JSON_METRICS_FILE, 'w') as ostr:
                ostr.write('[\n')
                for i in range(len(run_dirs)):
                    with open(osp.join(run_dirs[i], YAML_REPORT_FILE)) as istr:
                        data = yaml.safe_load(istr)
                        data.pop('category', None)
                        data.pop('command', None)
                        data['id'] = run_dirs[i]
                        json.dump(data, ostr, indent=2)
                    if i != len(run_dirs) - 1:
                        ostr.write(',')
                    ostr.write('\n')
                ostr.write(']\n')
494917
def process_file_object(file_obj, importer, progress):
    """"""

    logging.info("Processing schedule data.")

    try:
        handler = XmlCallbacks(importer, progress)
        parser = sax.make_parser()
        parser.setContentHandler(handler)
        parser.setErrorHandler(handler)
        parser.parse(file_obj)
    except:
        logging.exception("Parse failed.")
        raise

    logging.info("Schedule data processed.")
560437
def parse_input_file(text, variables=None):
    
    text = find_includes(text)
    lines = text.splitlines()
    tasks, linenumbers = find_tasks(lines)
    preamble = [line for line in lines[:linenumbers[0]]]
    logging.debug("Preamble:\n{}".format("\n".join(preamble)))
    if variables is not None:
        preamble += "\n" + "\n".join(variables)
    environment = create_environment(preamble)
    code_sections = []
    for n in range(len(linenumbers) - 1):
        code_sections.append((linenumbers[n], linenumbers[n+1]))
    for n, task in zip(code_sections, tasks):
        task["code"] = lines[n[0]: n[1]]
        task["environment"] = environment
    clean_tasks = []
    for task in tasks:
        clean_tasks.append(Task(**task))
    return clean_tasks
1132508
function displayMetrics(file) {
    var prefix = '';

    var code = fs.readFileSync(file, 'utf-8');

    // We remove any shebang at the start of the file since that isn't valid
    // Javascript and will trip up the parser.

    code = code.replace(/^#!.*\n/, '');

    var metrics = complexity(code);

    // Walk the metrics structure and add each member to the table rows.

    function walk(data, name) {
        // Fade out anonymous functions.

        if (name.indexOf('anon@') === 0) {
            name = name.grey;
        }

        rows.push([ prefix + name, data.ecc, data.arity, data.codeLines, data.commentLines, Math.round(100 * data.commentLines / data.codeLines) ]);

        // Add two spaces to the prefix before the next depth, to illustrate
        // the hierarchy in the table.

        prefix += '  ';
        _.each(data.children, walk);
        prefix = prefix.slice(0,prefix.length-2);
    }
    walk(metrics, path.basename(file).blue.bold);
}
904854
def parse_file(self, cu, analysis):
        """"""
        if hasattr(analysis, 'parser'):
            filename = cu.file_locator.relative_filename(cu.filename)
            source_lines = analysis.parser.lines
            with cu.source_file() as source_file:
                source = source_file.read()

            try:
                if sys.version_info < (3, 0):
                    encoding = source_encoding(source)
                    if encoding != 'utf-8':
                        source = source.decode(encoding).encode('utf-8')
            except UnicodeDecodeError:
                log.warning(
                    'Source file %s can not be properly decoded, skipping. '
                    'Please check if encoding declaration is ok',
                    os.path.basename(cu.filename))
                return
        else:
            if hasattr(cu, 'relative_filename'):
                filename = cu.relative_filename()
            else:
                filename = analysis.coverage.file_locator.relative_filename(
                    cu.filename)

            token_lines = analysis.file_reporter.source_token_lines()
            source_lines = list(enumerate(token_lines))
            source = analysis.file_reporter.source()

        coverage_lines = [self.get_hits(i, analysis)
                          for i in range(1, len(source_lines) + 1)]

        # ensure results are properly merged between platforms
        posix_filename = filename.replace(os.path.sep, '/')

        results = {
            'name': posix_filename,
            'source': source,
            'coverage': coverage_lines,
        }

        branches = self.get_arcs(analysis)
        if branches:
            results['branches'] = branches

        self.source_files.append(results)
327649
def parse(ctx, endpoint, check, here):
    """"""
    if here:
        output_dir = os.getcwd()
    else:
        output_dir = path_join(get_root(), check)
        if not dir_exists(output_dir):
            abort(
                'Check `{check}` does not exist; try `ddev create{repo_flag} {check}`.'.format(
                    check=check, repo_flag=' -e' if ctx.obj['repo_choice'] == 'extras' else ''
                )
            )

    endpoint = sanitize_endpoint(endpoint)

    echo_waiting('Scraping `{}`...'.format(endpoint))
    metrics = parse_metrics(endpoint)
    num_metrics = len(metrics)

    echo_success('\nGlobally available options:')
    echo_info('    t - Append .total to the available options')
    echo_info('    s - Skip')
    echo_info('    q - Quit')

    for i, (metric, data) in enumerate(sorted(iteritems(metrics)), 1):
        metric_parts = metric.split('_')
        metric_template = '{}'.join(metric_parts)
        num_separators = len(metric_parts) - 1

        metric_options = [
            metric_template.format(*possible_separators)
            for possible_separators in product(METRIC_SEPARATORS, repeat=num_separators)
        ]
        num_options = len(metric_options)

        default_option = num_options
        options_prompt = 'Choose an option (default {}, as-is): '.format(default_option)
        options_text = get_options_text(metric_options)

        finished = False
        choice_error = ''
        progress_status = '({} of {}) '.format(i, num_metrics)
        indent = ' ' * len(progress_status)

        while not finished:
            echo_success('\n{}{}'.format(progress_status, metric))

            echo_success('Type: ', nl=False, indent=indent)
            echo_info(data.get('type', 'None'))

            echo_success('Info: ', nl=False, indent=indent)
            echo_info(data.get('description', 'None'))

            echo_info(options_text)

            if choice_error:
                echo_warning(choice_error)

            echo_waiting(options_prompt, nl=False)

            if num_options >= 9:
                choice = ensure_unicode(input())
            else:
                # Terminals are odd and sometimes produce an erroneous null byte
                choice = '\x00'
                while choice == '\x00':
                    choice = click.getchar().strip()
                    try:
                        choice = ensure_unicode(choice)
                    except UnicodeDecodeError:
                        choice = repr(choice)

            if not choice:
                choice = default_option

            if choice == 't':
                echo_info('Append .total')
                for n in range(num_options):
                    metric_options[n] += '.total'
                options_text = get_options_text(metric_options)
                continue
            elif choice == 's':
                echo_info('Skip')
                echo_info('Skipped {}'.format(metric))
                break
            elif choice == 'q':
                echo_info('Exit')
                echo_warning('Exited at {}'.format(metric))
                return

            try:
                choice = int(choice)
            except Exception:
                pass

            if choice not in range(1, num_options + 1):
                echo_info(u'{}'.format(choice))
                choice_error = u'`{}` is not a valid option.'.format(choice)
                continue
            else:
                choice_error = ''

            option = metric_options[choice - 1]
            echo_info(option)

            data['dd_name'] = option

            finished = True

    metadata_file = path_join(output_dir, 'metadata.csv')
    echo_waiting('\nWriting `{}`... '.format(metadata_file), nl=False)

    metric_items = sorted(iteritems(metrics), key=lambda item: item[1]['dd_name'])
    output_lines = ['{}\n'.format(METADATA_CSV_HEADER)]
    for _, data in metric_items:
        metric_name = data['dd_name']
        metric_type = TYPE_MAP.get(data.get('type'), '')
        metric_description = data.get('description', '')
        if ',' in metric_description:
            metric_description = '"{}"'.format(metric_description)

        output_lines.append(
            '{check}.{metric_name},{metric_type},,,,{metric_description},0,{check},\n'.format(
                check=check, metric_name=metric_name, metric_type=metric_type, metric_description=metric_description
            )
        )

    write_file_lines(metadata_file, output_lines)
    echo_success('success!')

    metric_map = (
        'METRIC_MAP = {{\n'
        '{}\n'
        '}}'.format('\n'.join("    '{}': '{}',".format(metric, data['dd_name']) for metric, data in metric_items))
    )

    pyperclip.copy(metric_map)
    echo_success('\nThe metric map has been copied to your clipboard, paste it to any file you want!')
567167
def _generate_metrics(bam_fname, config_file, ref_file,
                      bait_file, target_file):
    
    with open(config_file) as in_handle:
        config = yaml.safe_load(in_handle)
    broad_runner = broad.runner_from_config(config)
    bam_fname = os.path.abspath(bam_fname)
    path = os.path.dirname(bam_fname)
    out_dir = os.path.join(path, "metrics")
    utils.safe_makedir(out_dir)
    with utils.chdir(out_dir):
        with tx_tmpdir() as tmp_dir:
            cur_bam = os.path.basename(bam_fname)
            if not os.path.exists(cur_bam):
                os.symlink(bam_fname, cur_bam)
            gen_metrics = PicardMetrics(broad_runner, tmp_dir)
            gen_metrics.report(cur_bam, ref_file,
                               _bam_is_paired(bam_fname),
                               bait_file, target_file)
    return out_dir
309705
function parser(file, callback) {
  // file exit conditions
  if(file.isNull()) {
    return callback(null, file); // pass along
  }

  if(file.isStream()) {
    return callback(new Error('Streaming not supported'));
  }

  try {

    var
      /** @type {string} */
      text     = String(file.contents.toString('utf8')),
      lines    = text.split('\n'),
      filename = file.path.substring(0, file.path.length - 4),
      key      = 'server/documents',
      position = filename.indexOf(key)
    ;

    // exit conditions
    if(!lines) {
      return;
    }
    if(position < 0) {
      return callback(null, file);
    }

    filename = filename.substring(position + key.length + 1, filename.length);

    var
      lineCount = lines.length,
      active    = false,
      yaml      = [],
      categories = [
        'UI Element',
        'UI Global',
        'UI Collection',
        'UI View',
        'UI Module',
        'UI Behavior'
      ],
      index,
      meta,
      line
    ;

    for(index = 0; index < lineCount; index++) {

      line = lines[index];

      // Wait for metadata block to begin
      if(!active) {
        if(startsWith(line, '---')) {
          active = true;
        }
        continue;
      }
      // End of metadata block, stop parsing.
      if(startsWith(line, '---')) {
        break;
      }
      yaml.push(line);
    }


    // Parse yaml.
    meta = YAML.parse(yaml.join('\n'));
    if(meta && meta.type && meta.title && inArray(meta.type, categories) ) {
      meta.category = meta.type;
      meta.filename = filename;
      meta.url      = '/' + filename;
      meta.title    = meta.title;
      // Primary key will by filepath
      data[meta.element] = meta;
    }
    else {
      // skip
      // console.log(meta);
    }


  }

  catch(error) {
    console.log(error, filename);
  }

  callback(null, file);

}
835954
function parseFile(fs, file, type) {
    var filepath = file.getPath();
    var parser = file.getParser();

    if (!parser) {
        return Promise.reject(
            error.FileNotParsableError({
                filename: filepath
            })
        );
    }

    return fs.readAsString(filepath)
    .then(function(content) {
        if (type === 'readme') {
            return parser.parseReadme(content);
        } else if (type === 'glossary') {
            return parser.parseGlossary(content);
        } else if (type === 'summary') {
            return parser.parseSummary(content);
        } else if (type === 'langs') {
            return parser.parseLanguages(content);
        } else {
            throw new Error('Parsing invalid type "' + type + '"');
        }
    })
    .then(function(result) {
        return [
            file,
            result
        ];
    });
}
31146
def collect(self):
        
        if not os.access(self.PROC, os.R_OK):
            return False

        # Open PROC file
        file = open(self.PROC, 'r')
        # Get data
        cpuCount = None
        for line in file:
            if not cpuCount:
                cpuCount = len(line.split())
            else:
                data = line.strip().split(None, cpuCount + 2)
                data[0] = data[0].replace(':', '')

                if len(data) == 2:
                    metric_name = data[0]
                    metric_value = data[1]
                    self.publish(metric_name,
                                 self.derivative(metric_name,
                                                 long(metric_value),
                                                 counter))
                else:
                    if len(data[0]) == cpuCount + 1:
                        metric_name = data[0] + '.'
                    elif len(data[0]) == 3:
                        metric_name = (
                            ((data[-2] + ' ' +
                              data[-1]).replace(' ', '_')) + '.')
                    else:
                        metric_name = (
                            ((data[-2]).replace(' ', '_')) +
                            '.' +
                            ((data[-1]).replace(', ', '-').replace(' ', '_')) +
                            '.' + data[0] + '.')
                    total = 0
                    for index, value in enumerate(data):
                        if index == 0 or index >= cpuCount + 1:
                            continue

                        metric_name_node = metric_name + 'CPU' + str(index - 1)
                        value = int(self.derivative(metric_name_node,
                                                    long(value), counter))
                        total += value
                        self.publish(metric_name_node, value)

                    # Roll up value
                    metric_name_node = metric_name + 'total'
                    self.publish(metric_name_node, total)

        # Close file
        file.close()
50335
def from_file(cls, fn, *args, **kwargs):
        
        return AmiraHeader(get_parsed_data(fn, *args, **kwargs))