Setup instance to extract metrics from the proper run

        :param outdir: run directory
        :param log_prefix: log filenames prefix
191496
def context(self, outdir, log_prefix):
        
        try:
            self._outdir = outdir
            self._log_prefix = log_prefix
            yield
        finally:
            self._log_prefix = None
            self._outdir = None
1073163
def prepare_outdir(outdir):
    
    if outdir:
        outdir = os.path.expanduser(outdir)
        if not os.path.isdir(outdir):
            try:
                os.makedirs(outdir)
            except os.error as e:
                raise JobExecutionError('Failed to create outdir "{}".\n{}'.format(outdir, str(e)))
562587
def setup_logdir(self):
    # todo: locking on logdir creation

    
    run_name = ncluster_globals.get_run_for_task(self)
    self.log("Creating logdir for run "+run_name)
    logdir_root = ncluster_globals.LOGDIR_ROOT
    assert logdir_root

    self.run(f'mkdir -p {logdir_root}')
    find_command = f'find {logdir_root} -maxdepth 1 -type d'

    stdout, stderr = self.run_with_output(find_command)
    logdir = f"{logdir_root}/{run_name}"

    counter = 0
    while logdir in stdout:
      counter += 1
      new_logdir = f'{logdir_root}/{run_name}.{counter:02d}'
      self.log(f'Warning, logdir {logdir} exists, deduping to {new_logdir}')
      logdir = new_logdir
    self.run(f'mkdir -p {logdir}')

    ncluster_globals.set_logdir(run_name, logdir)
    return logdir
572707
def log_root(self):
        
        var_log = (
            os.path.join(sys.prefix, 'var', 'log')
            .replace('/usr/var', '/var')
        )
        if not os.path.isdir(var_log):
            os.makedirs(var_log)
        return var_log
1154494
public static void incrLogMetrics(Map<String, Long> incrMetrics) {
    if (incrMetrics == null || incrMetrics.size() == 0) {
      return;
    }
    MetricsRegistry registry = RaidNodeMetrics.getInstance(
        RaidNodeMetrics.DEFAULT_NAMESPACE_ID).getMetricsRegistry();
    Map<String, MetricsTimeVaryingLong> logMetrics = RaidNodeMetrics.getInstance(
        RaidNodeMetrics.DEFAULT_NAMESPACE_ID).logMetrics;
    synchronized(logMetrics) {
      for (String key : incrMetrics.keySet()) {
        if (!logMetrics.containsKey(key)) {
          logMetrics.put(key, new MetricsTimeVaryingLong(key, registry));
        }
        ((MetricsTimeVaryingLong)logMetrics.get(key)).inc(incrMetrics.get(key));
      }
    }
  }
846453
def log(env, status, header, began_at)
      now    = Time.now
      length = extract_content_length(header)

      msg = Hash[
        http:    env[HTTP_VERSION],
        verb:    env[REQUEST_METHOD],
        status:  status.to_s[0..3],
        ip:      env[HTTP_X_FORWARDED_FOR] || env[REMOTE_ADDR],
        path:    env[SCRIPT_NAME] + env[PATH_INFO].to_s,
        length:  length,
        params:  extract_params(env),
        elapsed: now - began_at
      ]

      logger = @logger || env[RACK_ERRORS]
      # Standard library logger doesn't support write but it supports << which actually
      # calls to write on the log device without formatting
      if logger.respond_to?(:write)
        logger.write(msg)
      else
        logger.info(msg)
      end
    end
1085634
def make_tmp_dir(prefix):
    

    # Time in ISO8601 format
    now = datetime.now().isoformat()

    # A random UUID is appended to the output directory in order to
    # avoid name collisions
    f_uuid = uuid.uuid4().hex

    # Directory name and path
    out_dir = "{out}/{p}-{now}-{uuid}".format(out=_output_dir, p=prefix,
                                              now=now, uuid=f_uuid)

    # create the actual directory
    if not _dry_run:
        os.makedirs(out_dir)
    log.msg_debug("Output directory: {out_dir}".format(out_dir=_output_dir))

    # Return a string containing the name and path of the output directory
    return out_dir
964765
private void runForEachK(String prefix, int mink, int maxk, IntFunction<OutlierResult> runner, BiConsumer<String, OutlierResult> out) {
    if(isDisabled(prefix)) {
      LOG.verbose("Skipping (disabled): " + prefix);
      return; // Disabled
    }
    LOG.verbose("Running " + prefix);
    final int digits = (int) FastMath.ceil(FastMath.log10(krange.getMax() + 1));
    final String format = "%s-%0" + digits + "d";
    krange.forEach(k -> {
      if(k >= mink && k <= maxk) {
        Duration time = LOG.newDuration(this.getClass().getCanonicalName() + "." + prefix + ".k" + k + ".runtime").begin();
        OutlierResult result = runner.apply(k);
        LOG.statistics(time.end());
        if(result != null) {
          out.accept(String.format(Locale.ROOT, format, prefix, k), result);
          result.getHierarchy().removeSubtree(result);
        }
      }
    });
  }
54076
def write_installed_files(self, paths, prefix, dry_run=False):
        
        prefix = os.path.join(prefix, '')
        base = os.path.dirname(self.path)
        base_under_prefix = base.startswith(prefix)
        base = os.path.join(base, '')
        record_path = self.get_distinfo_file('RECORD')
        logger.info('creating %s', record_path)
        if dry_run:
            return None
        with CSVWriter(record_path) as writer:
            for path in paths:
                if os.path.isdir(path) or path.endswith(('.pyc', '.pyo')):
                    # do not put size and hash, as in PEP-376
                    hash_value = size = ''
                else:
                    size = '%d' % os.path.getsize(path)
                    with open(path, 'rb') as fp:
                        hash_value = self.get_hash(fp.read())
                if path.startswith(base) or (base_under_prefix and
                                             path.startswith(prefix)):
                    path = os.path.relpath(path, base)
                writer.writerow((path, hash_value, size))

            # add the RECORD file itself
            if record_path.startswith(base):
                record_path = os.path.relpath(record_path, base)
            writer.writerow((record_path, '', ''))
        return record_path
846605
def dump_data(logdir):
  """"""
  # Create a tfevents file in the logdir so it is detected as a run.
  write_empty_event_file(logdir)

  plugin_logdir = plugin_asset_util.PluginDirectory(
      logdir, profile_plugin.ProfilePlugin.plugin_name)
  _maybe_create_directory(plugin_logdir)

  for run in profile_demo_data.RUNS:
    run_dir = os.path.join(plugin_logdir, run)
    _maybe_create_directory(run_dir)
    if run in profile_demo_data.TRACES:
      with open(os.path.join(run_dir, 'trace'), 'w') as f:
        proto = trace_events_pb2.Trace()
        text_format.Merge(profile_demo_data.TRACES[run], proto)
        f.write(proto.SerializeToString())

    if run not in profile_demo_data.TRACE_ONLY:
      shutil.copyfile('tensorboard/plugins/profile/profile_demo.op_profile.json',
                      os.path.join(run_dir, 'op_profile.json'))
      shutil.copyfile(
          'tensorboard/plugins/profile/profile_demo.memory_viewer.json',
          os.path.join(run_dir, 'memory_viewer.json'))
      shutil.copyfile(
          'tensorboard/plugins/profile/profile_demo.pod_viewer.json',
          os.path.join(run_dir, 'pod_viewer.json'))
      shutil.copyfile(
          'tensorboard/plugins/profile/profile_demo.google_chart_demo.json',
          os.path.join(run_dir, 'google_chart_demo.json'))

  # Unsupported tool data should not be displayed.
  run_dir = os.path.join(plugin_logdir, 'empty')
  _maybe_create_directory(run_dir)
  with open(os.path.join(run_dir, 'unsupported'), 'w') as f:
    f.write('unsupported data')
851412
def run_all(logdir, verbose=False):
  
  for initial_temperature in [270.0, 310.0, 350.0]:
    for final_temperature in [270.0, 310.0, 350.0]:
      for heat_coefficient in [0.001, 0.005]:
        run_name = 'temperature:t0=%g,tA=%g,kH=%g' % (
            initial_temperature, final_temperature, heat_coefficient)
        if verbose:
          print('--- Running: %s' % run_name)
        run(logdir, run_name,
            initial_temperature, final_temperature, heat_coefficient)
568196
def _init_file_logger(logger, level, log_path, log_size, log_count):
    
    if level not in [logging.NOTSET, logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL]:
        level = logging.DEBUG

    for h in logger.handlers:
        if isinstance(h, logging.handlers.RotatingFileHandler):
            if h.level == level:
                return

    fh = logging.handlers.RotatingFileHandler(
        log_path, maxBytes=log_size, backupCount=log_count)
    fh.setLevel(level)
    fh.setFormatter(_formatter)
    logger.addHandler(fh)
4182
def setup_logging(default_path='logging.yaml', env_key='LOG_CFG'):
    
    path = default_path
    value = os.getenv(env_key, None)
    if value:
        path = value
    if os.path.exists(path):
        with open(path, 'rt') as f:
            configs = yaml.safe_load(f.read())
        logging.config.dictConfig(configs)
    else:
        logging.config.dictConfig(config)
567167
def _generate_metrics(bam_fname, config_file, ref_file,
                      bait_file, target_file):
    
    with open(config_file) as in_handle:
        config = yaml.safe_load(in_handle)
    broad_runner = broad.runner_from_config(config)
    bam_fname = os.path.abspath(bam_fname)
    path = os.path.dirname(bam_fname)
    out_dir = os.path.join(path, "metrics")
    utils.safe_makedir(out_dir)
    with utils.chdir(out_dir):
        with tx_tmpdir() as tmp_dir:
            cur_bam = os.path.basename(bam_fname)
            if not os.path.exists(cur_bam):
                os.symlink(bam_fname, cur_bam)
            gen_metrics = PicardMetrics(broad_runner, tmp_dir)
            gen_metrics.report(cur_bam, ref_file,
                               _bam_is_paired(bam_fname),
                               bait_file, target_file)
    return out_dir
185280
def _teardown_log_prefix(self):
        """"""
        self._logger_console_fmtter.prefix = ''
        self._logger_console_fmtter.plugin_id = ''
        self._logger_file_fmtter.prefix = ' '
        self._logger_file_fmtter.plugin_id = ''
1208339
def write_metrics_file(metrics: List[Dict[str, Any]], path: str):
    
    with open(path, 'w') as metrics_out:
        for checkpoint, metric_dict in enumerate(metrics, 1):
            metrics_str = "\t".join(["{}={}".format(name, value) for name, value in sorted(metric_dict.items())])
            metrics_out.write("{}\t{}\n".format(checkpoint, metrics_str))
1314343
def setup_logging(logfile_path):
    
    date_fmt = "%y-%m-%d (%a) %H:%M:%S"

    std_formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s',
                                      date_fmt)

    # Get to the following size before splitting log into multiple files:
    log_chunk_bytesize = 5e6

    info_logfile = logging.handlers.RotatingFileHandler(logfile_path,
                                                        maxBytes=log_chunk_bytesize,
                                                        backupCount=10)
    info_logfile.setFormatter(std_formatter)
    info_logfile.setLevel(logging.DEBUG)

    stdout_log = logging.StreamHandler()
    stdout_log.setLevel(logging.DEBUG)
    stdout_log.setFormatter(std_formatter)

    # Set up root logger
    logger = logging.getLogger()
    logger.handlers = []
    logger.setLevel(logging.DEBUG)
    logger.addHandler(info_logfile)
    logger.addHandler(stdout_log)
    logging.getLogger('iso8601').setLevel(
        logging.ERROR)  # Suppress iso8601 debug log.
    return logger