Creates scene metrics with information obtained from the deployed
config file.
197097
public static MisoSceneMetrics getSceneMetrics ()
    {
        return new MisoSceneMetrics(
            config.getValue(TILE_WIDTH_KEY, DEF_TILE_WIDTH),
            config.getValue(TILE_HEIGHT_KEY, DEF_TILE_HEIGHT),
            config.getValue(FINE_GRAN_KEY, DEF_FINE_GRAN));
    }
273373
private void loadConfigFile(String configFile) {
    try (InputStream is = new FileInputStream(configFile)) {
      mProperties.load(is);
    } catch (Exception e) {
      LOG.error("Error loading metrics configuration file.", e);
    }
  }
60779
def __create_heatmap_plot(self):
        """"""
        # Define the heatmap plot.
        height = len(self.pyfile.lines) / 3
        width = max(map(lambda x: len(x), self.pyfile.lines)) / 8
        self.fig, self.ax = plt.subplots(figsize=(width, height))

        # Set second sub plot to occupy bottom 20%
        plt.subplots_adjust(bottom=0.20)

        # Heat scale orange to red
        heatmap = self.ax.pcolor(self.pyfile.data, cmap="OrRd")

        # X Axis
        # Remove X axis.
        self.ax.xaxis.set_visible(False)

        # Y Axis
        # Create lables for y-axis ticks
        row_labels = range(1, self.pyfile.length + 1)
        # Set y-tick labels.
        self.ax.set_yticklabels(row_labels, minor=False)
        # Put y-axis major ticks at the middle of each cell.
        self.ax.set_yticks(np.arange(self.pyfile.data.shape[0]) + 0.5, minor=False)
        # Inver y-axis to have top down line numbers
        self.ax.invert_yaxis()

        # Plot definitions
        # Set plot y-axis label.
        plt.ylabel("Line Number")
        # Annotate each cell with lines in file in order.
        max_time_spent_on_a_line = max(self.pyfile.data)
        for i, line in enumerate(self.pyfile.lines):
            # In order to ensure easy readability of the code, we need to
            # invert colour of text display for darker colours which
            # correspond to higher amount of time spent on the line.
            if self.pyfile.data[i] >= 0.7 * max_time_spent_on_a_line:
                color = (1.0, 1.0, 1.0)  # White text
            else:
                color = (0.0, 0.0, 0.0)  # Black text
            plt.text(
                0.0,
                i + 0.5,
                line,
                ha="left",
                va="center",
                color=color,
                clip_on=True,
            )

        # Define legend
        cbar = plt.colorbar(heatmap)
        cbar.set_label("# of seconds")
903248
def _setup_metrics(self):
        
        path = os.environ.get("prometheus_multiproc_dir")
        if not os.path.exists(self.metrics_dir):
            try:
                log.info("Creating metrics directory")
                os.makedirs(self.metrics_dir)
            except OSError:
                log.error("Failed to create metrics directory!")
                raise ConfigurationException("Failed to create metrics directory!")
            path = self.metrics_dir
        elif path != self.metrics_dir:
            path = self.metrics_dir
        os.environ['prometheus_multiproc_dir'] = path
        log.info("Cleaning metrics collection directory")
        log.debug("Metrics directory set to: {}".format(path))
        files = os.listdir(path)
        for f in files:
            if f.endswith(".db"):
                os.remove(os.path.join(path, f))
            log.debug("Starting metrics exposition")
        if self.metrics_enabled:
            registry = CollectorRegistry()
            multiprocess.MultiProcessCollector(registry)
            start_http_server(
                port=self.metrics_port,
                addr=self.metrics_address,
                registry=registry
            )
310792
def configs(self):
    """"""
    run_path_pairs = list(self.run_paths.items())
    self._append_plugin_asset_directories(run_path_pairs)
    # If there are no summary event files, the projector should still work,
    # treating the `logdir` as the model checkpoint directory.
    if not run_path_pairs:
      run_path_pairs.append(('.', self.logdir))
    if (self._run_paths_changed() or
        _latest_checkpoints_changed(self._configs, run_path_pairs)):
      self.readers = {}
      self._configs, self.config_fpaths = self._read_latest_config_files(
          run_path_pairs)
      self._augment_configs_with_checkpoint_info()
    return self._configs
563697
def main():
    
    logging.basicConfig(level=logging.INFO)

    run_metrics = py_interop_run_metrics.run_metrics()
    summary = py_interop_summary.run_summary()

    valid_to_load = py_interop_run.uchar_vector(py_interop_run.MetricCount, 0)
    py_interop_run_metrics.list_summary_metrics_to_load(valid_to_load)

    for run_folder_path in sys.argv[1:]:
        run_folder = os.path.basename(run_folder_path)
        try:
            run_metrics.read(run_folder_path, valid_to_load)
        except Exception, ex:
            logging.warn("Skipping - cannot read RunInfo.xml: %s - %s"%(run_folder, str(ex)))
            continue
        py_interop_summary.summarize_run_metrics(run_metrics, summary)
        error_rate_read_lane_surface = numpy.zeros((summary.size(), summary.lane_count(), summary.surface_count()))
        for read_index in xrange(summary.size()):
            for lane_index in xrange(summary.lane_count()):
                for surface_index in xrange(summary.surface_count()):
                    error_rate_read_lane_surface[read_index, lane_index, surface_index] = \
                        summary.at(read_index).at(lane_index).at(surface_index).error_rate().mean()
        logging.info("Run Folder: "+run_folder)
        for read_index in xrange(summary.size()):
            read_summary = summary.at(read_index)
            logging.info("Read "+str(read_summary.read().number())+" - Top Surface Mean Error: "+str(error_rate_read_lane_surface[read_index, :, 0].mean()))
739849
private void populateDatanodeMetrics(ArrayList<DatanodeDescriptor> live,
      ArrayList<DatanodeDescriptor> dead) {

    DatanodeStatus status = FSNamesystemDatanodeHelper.getDatanodeStats(fsNameSystem,
        live, dead);

    // populate metrics
    numLiveNodes.set(status.numLive);
    numLiveExcludedNodes.set(status.numLiveExcluded);
    numLiveDecommissioningInProgressNodes.set(status.numLiveDecommissioningInProgress);
    numLiveDecommissioned.set(status.numLiveDecommissioned);

    numDeadNodes.set(status.numDead);
    numDeadExcludedNodes.set(status.numDeadExcluded);
    numDeadDecommissioningNotCompletedNodes.set(status.numDeadDecommissioningNotCompleted);
    numDeadDecommissioned.set(status.numDeadDecommissioned);
  }
1124513
def compute_file_metrics(processors, language, key, token_list):
    """"""
    # multiply iterator
    tli = itertools.tee(token_list, len(processors))
    metrics = OrderedDict()

    # reset all processors
    for p in processors:
        p.reset()

    # process all tokens
    for p, tl in zip(processors, tli):
        p.process_file(language, key, tl)

    # collect metrics from all processors
    for p in processors:
        metrics.update(p.metrics)

    return metrics
1088078
def _create_imshow_objects(self):
        """"""

        # uniform values for initial image can cause weird behaviour with normalization
        #       as imshow.set_data() does not automatically update the normalization!!
        # using random data is a better choice
        random_image = np.random.rand(20, 20)
        self.images = [None] * len(self.flat_grid)
        for ix, ax in enumerate(self.flat_grid):
            self.images[ix] = ax.imshow(random_image, **self.display_params)
1224481
def _createAbsMagEstimationDict():
    
    magnitude_estimation_filepath = resource_filename(
        __name__, 'data/magnitude_estimation.dat')
    raw_table = np.loadtxt(magnitude_estimation_filepath, '|S5')

    absMagDict = {
        'O': {},
        'B': {},
        'A': {},
        'F': {},
        'G': {},
        'K': {},
        'M': {}}
    for row in raw_table:
        if sys.hexversion >= 0x03000000:
            # otherwise we get byte ints or b' caused by 2to3
            starClass = row[0].decode("utf-8")
            absMagDict[starClass[0]][int(starClass[1])] = [
                float(x) for x in row[1:]]
        else:
            # dict of spectral type = {abs mag for each luminosity class}
            absMagDict[row[0][0]][int(row[0][1])] = [float(x) for x in row[1:]]

    # manually typed from table headers - used to match columns with the L
    # class (header)
    LClassRef = {
        'V': 0,
        'IV': 1,
        'III': 2,
        'II': 3,
        'Ib': 4,
        'Iab': 5,
        'Ia': 6,
        'Ia0': 7}

    return absMagDict, LClassRef
898654
def publish_scene_config(self, scene_id, config):
        """"""
        self.sequence_number += 1
        self.publisher.send_multipart(msgs.MessageBuilder.scene_config(self.sequence_number, scene_id, config))
        return self.sequence_number
725473
def gen_sls_config_files(stage, region):
    """"""
    names = []
    for ext in ['yml', 'json']:
        # Give preference to explicit stage-region files
        names.append(
            os.path.join('env',
                         "%s-%s.%s" % (stage, region, ext))
        )
        names.append("config-%s-%s.%s" % (stage, region, ext))
        # Fallback to stage name only
        names.append(
            os.path.join('env',
                         "%s.%s" % (stage, ext))
        )
        names.append("config-%s.%s" % (stage, ext))
    return names
895937
def metrics(self) -> list:
        
        my_metrics = [
            FramesMetric("frames"),
            FPSMetric("fps"),
            EpisodeRewardMetric('PMM:episode_rewards'),
            EpisodeRewardMetricQuantile('P09:episode_rewards', quantile=0.9),
            EpisodeRewardMetricQuantile('P01:episode_rewards', quantile=0.1),
            EpisodeLengthMetric("episode_length")
        ]

        return my_metrics + self.algo.metrics() + self.env_roller.metrics()
1078093
def _create_graphical_report(inputfile, pwm, background, closest_match, outdir, stats, best_id=None):
    """"""
    if best_id is None:
        best_id = {}

    logger.debug("Creating graphical report")
    
    class ReportMotif(object):
        """Placeholder for motif stats."""
        pass

    config = MotifConfig()
    
    imgdir = os.path.join(outdir, "images")
    if not os.path.exists(imgdir):
        os.mkdir(imgdir)
    
    motifs = read_motifs(pwm, fmt="pwm")
    
    roc_img_file = "%s_roc.%s"

    dbpwm = config.get_default_params()["motif_db"]
    pwmdir = config.get_motif_dir()

    dbmotifs = read_motifs(os.path.join(pwmdir, dbpwm), as_dict=True)
    
    report_motifs = []
    for motif in motifs:
        
        rm = ReportMotif()
        rm.id = motif.id
        rm.id_href = {"href": "#%s" % motif.id}
        rm.id_name = {"name": motif.id}
        rm.img = {"src":  os.path.join("images", "%s.png" % motif.id)}
        motif.to_img(os.path.join(outdir, "images/{}.png".format(motif.id)), fmt="PNG")
        
        # TODO: fix best ID
        rm.best = "Gimme"#best_id[motif.id]

        rm.consensus = motif.to_consensus()
        rm.stars = int(np.mean(
                [stats[str(motif)][bg].get("stars", 0) for bg in background]
                ) + 0.5)

        rm.bg = {}
        for bg in background:
            rm.bg[bg] = {}
            this_stats = stats.get(str(motif), {}).get(bg)
            # TODO: fix these stats
            rm.bg[bg]["e"] = "%0.2f" % this_stats.get("enr_at_fpr", 1.0)
            rm.bg[bg]["p"] = "%0.2f" % this_stats.get("phyper_at_fpr", 1.0)
            rm.bg[bg]["auc"] = "%0.3f" % this_stats.get("roc_auc", 0.5)
            rm.bg[bg]["mncp"] = "%0.3f" % this_stats.get("mncp", 1.0)
            rm.bg[bg]["roc_img"] = {"src": "images/" + os.path.basename(roc_img_file % (motif.id, bg)) + ".png"}
            rm.bg[bg][u"roc_img_link"] = {u"href": "images/" + os.path.basename(roc_img_file % (motif.id, bg)) + ".png"}

        rm.histogram_img = {"data":"images/%s_histogram.svg" % motif.id}
        rm.histogram_link= {"href":"images/%s_histogram.svg" % motif.id}
        
        match_id = closest_match[motif.id][0]
        dbmotifs[match_id].to_img(os.path.join(outdir, "images/{}.png".format(match_id)), fmt="PNG")
    
        rm.match_img = {"src":  "images/{}.png".format(match_id)}
        rm.match_id = closest_match[motif.id][0]
        rm.match_pval = "%0.2e" % closest_match[motif.id][1][-1]

        report_motifs.append(rm)
    
    total_report = os.path.join(outdir, "motif_report.html")

    star_img = os.path.join(config.get_template_dir(), "star.png")
    shutil.copyfile(star_img, os.path.join(outdir, "images", "star.png"))

    env = jinja2.Environment(loader=jinja2.FileSystemLoader([config.get_template_dir()]))
    template = env.get_template("report_template.jinja.html")
    # TODO: title
    result = template.render(
                    motifs=report_motifs, 
                    inputfile=inputfile, 
                    date=datetime.today().strftime("%d/%m/%Y"), 
                    version=__version__,
                    bg_types=list(background.keys()))

    with open(total_report, "wb") as f:
        f.write(result.encode('utf-8'))
443262
def get_custom_metrics():
    
    _rse = mx.metric.create(rse)
    _rae = mx.metric.create(rae)
    _corr = mx.metric.create(corr)
    return mx.metric.create([_rae, _rse, _corr])
983292
def scene_add(frames):
        """"""
        reader = MessageReader(frames)
        results = reader.string("command").uint32("animation_id").string("name").uint8_3("color").uint32("velocity").string("config").get()
        if results.command != "scene.add":
            raise MessageParserError("Command is not 'scene.add'")
        return (results.animation_id, results.name, np.array([results.color[0]/255, results.color[1]/255, results.color[2]/255]),
                results.velocity/1000, results.config)
1075379
def diff(ctx, files, metrics, all, detail):
    """"""
    config = ctx.obj["CONFIG"]

    if not exists(config):
        handle_no_cache(ctx)

    if not metrics:
        metrics = get_default_metrics(config)
        logger.info(f"Using default metrics {metrics}")
    else:
        metrics = metrics.split(",")
        logger.info(f"Using specified metrics {metrics}")

    from wily.commands.diff import diff

    logger.debug(f"Running diff on {files} for metric {metrics}")
    diff(
        config=config, files=files, metrics=metrics, changes_only=not all, detail=detail
    )