Create a run config.

  Args:
    hp: model hyperparameters
    output_dir: model's output directory, defaults to output_dir flag.

  Returns:
    a run config
421353
def create_run_config(hp, output_dir=None):
  
  save_ckpt_steps = max(FLAGS.iterations_per_loop, FLAGS.local_eval_frequency)
  save_ckpt_secs = FLAGS.save_checkpoints_secs or None
  if save_ckpt_secs:
    save_ckpt_steps = None
  assert FLAGS.output_dir or FLAGS.checkpoint_path
  tpu_config_extra_kwargs = {}
  if FLAGS.tpu_job_name is not None:
    tpu_config_extra_kwargs["tpu_job_name"] = FLAGS.tpu_job_name

  if getattr(hp, "mtf_mode", False):
    save_ckpt_steps = None  # Disable the default saver
    save_ckpt_secs = None  # Disable the default saver
    tpu_config_extra_kwargs = {
        "num_cores_per_replica": 1,
        "per_host_input_for_training": tpu_config.InputPipelineConfig.BROADCAST,
    }

  # the various custom getters we have written do not play well together yet.
  # TODO(noam): ask rsepassi for help here.
  daisy_chain_variables = (
      hp.daisy_chain_variables and
      hp.activation_dtype == "float32" and
      hp.weight_dtype == "float32")
  return trainer_lib.create_run_config(
      model_name=FLAGS.model,
      model_dir=output_dir or os.path.expanduser(FLAGS.output_dir),
      master=FLAGS.master,
      iterations_per_loop=FLAGS.iterations_per_loop,
      num_shards=FLAGS.tpu_num_shards,
      log_device_placement=FLAGS.log_device_placement,
      save_checkpoints_steps=save_ckpt_steps,
      save_checkpoints_secs=save_ckpt_secs,
      keep_checkpoint_max=FLAGS.keep_checkpoint_max,
      keep_checkpoint_every_n_hours=FLAGS.keep_checkpoint_every_n_hours,
      num_gpus=FLAGS.worker_gpu,
      gpu_order=FLAGS.gpu_order,
      num_async_replicas=FLAGS.worker_replicas,
      gpu_mem_fraction=FLAGS.worker_gpu_memory_fraction,
      enable_graph_rewriter=FLAGS.enable_graph_rewriter,
      use_tpu=FLAGS.use_tpu,
      use_tpu_estimator=FLAGS.use_tpu_estimator,
      xla_jit_level=FLAGS.xla_jit_level,
      schedule=FLAGS.schedule,
      no_data_parallelism=hp.no_data_parallelism,
      optionally_use_dist_strat=FLAGS.optionally_use_dist_strat,
      daisy_chain_variables=daisy_chain_variables,
      ps_replicas=FLAGS.ps_replicas,
      ps_job=FLAGS.ps_job,
      ps_gpu=FLAGS.ps_gpu,
      sync=FLAGS.sync,
      worker_id=FLAGS.worker_id,
      worker_job=FLAGS.worker_job,
      random_seed=FLAGS.random_seed,
      tpu_infeed_sleep_secs=FLAGS.tpu_infeed_sleep_secs,
      inter_op_parallelism_threads=FLAGS.inter_op_parallelism_threads,
      log_step_count_steps=FLAGS.log_step_count_steps,
      intra_op_parallelism_threads=FLAGS.intra_op_parallelism_threads,
      tpu_config_extra_kwargs=tpu_config_extra_kwargs,
      cloud_tpu_name=FLAGS.cloud_tpu_name)
903202
def output(results_dir, config, parent='../../'):
    
    start = time.time()
    print("Compiling results...")
    results_dir = os.path.abspath(results_dir)
    results = ReportResults(config['run_time'], config['results_ts_interval'])
    results.compile_results()
    print("Results compiled in {} seconds\n".format(time.time() - start))

    if results.total_transactions == 0:
        print("No results, cannot create report")
        return False

    print_infos(results)

    data = {
        'report': results,
        'run_time': config['run_time'],
        'ts_interval': config['results_ts_interval'],
        'turrets_config': results.turrets,
        'results': {"all": results.main_results, "timers": results.timers_results}
    }

    print("Generating graphs...")
    partial = time.time()
    generate_graphs(results.main_results, 'All_Transactions', results_dir)

    for key, value in results.timers_results.items():
        generate_graphs(value, key, results_dir)
    print("All graphs generated in {} seconds\n".format(time.time() - partial))

    write_template(data, results_dir, parent)
    print("Full report generated in {} seconds".format(time.time() - start))
    return True
296264
def output_dir(self):
        
        if self._output_dir is None:
            self._output_dir = self._get_output_dir()
        return self._output_dir
1201890
def _create_output_from_match(self, match_result):
        
        if isinstance(match_result, dict):
            return LinterOutput(self.name, **match_result)
        return LinterOutput(self.name, *match_result)
442324
def output_filename(output_dir, key_handle, public_id):
    
    parts = [output_dir, key_handle] + pyhsm.util.group(public_id, 2)
    path = os.path.join(*parts)

    if not os.path.isdir(path):
        os.makedirs(path)

    return os.path.join(path, public_id)
1171800
def run(self, **kwargs):
        
        from calculate import compute
        self.app_main(**kwargs)

        # get the default output name
        output = osp.join(self.exp_config['expdir'], 'output.dat')

        # save the paths in the configuration
        self.exp_config['output'] = output

        # run the model
        data = np.loadtxt(self.exp_config['infile'])
        out = compute(data)
        # save the output
        self.logger.info('Saving output data to %s', osp.relpath(output))
        np.savetxt(output, out)

        # store some additional information in the configuration of the
        # experiment
        self.exp_config['mean'] = mean = float(out.mean())
        self.exp_config['std'] = std = float(out.std())
        self.logger.debug('Mean: %s, Standard deviation: %s', mean, std)
482956
def create_teacher_experiment(run_config, hparams, argv):
  """"""
  tf.logging.info("training teacher")
  tf.logging.set_verbosity(tf.logging.INFO)
  trainer_lib.set_random_seed(FLAGS.random_seed)
  usr_dir.import_usr_dir(FLAGS.t2t_usr_dir)
  t2t_trainer.maybe_log_registry_and_exit()

  if FLAGS.cloud_mlengine:
    return cloud_mlengine.launch()

  if FLAGS.generate_data:
    t2t_trainer.generate_data()

  if cloud_mlengine.job_dir():
    FLAGS.output_dir = cloud_mlengine.job_dir()

  if argv:
    t2t_trainer.set_hparams_from_args(argv[1:])

  hparams.distill_phase = "train"
  exp_fn = t2t_trainer.create_experiment_fn()
  exp = exp_fn(run_config, hparams)
  return exp
182201
def add_output_arg(self, out):
        
        self.add_arg(out._dax_repr())
        self._add_output(out)
1298832
def runprofile(mainfunction, output, timeout = 0, calibrate=False):
    '''
    
    '''
    if noprofiler == True:
            print('ERROR: profiler and/or pstats library missing ! Please install it (probably package named python-profile) before running a profiling !')
            return False
    # This is the main function for profiling
    def _profile():
        profile.run(mainfunction, output)
    print('=> RUNNING FUNCTIONS PROFILER\n\n'); sys.stdout.flush();
    # Calibrate the profiler (only use this if the profiler produces some funny stuff, but calibration can also produce even more funny stuff with the latest cProfile of Python v2.7! So you should only enable calibration if necessary)
    if calibrate:
        print('Calibrating the profiler...'); sys.stdout.flush();
        cval = calibrateprofile()
        print('Calibration found value : %s' % cval); sys.stdout.flush();
    print('Initializing the profiler...'); sys.stdout.flush();
    # Run in timeout mode (if the function cannot ends by itself, this is the best mode: the function must ends for the profile to be saved)
    if timeout > 0:
        pthread = KThread(target=_profile) # we open the function with the profiler, in a special killable thread (see below why)
        print('Will now run the profiling and terminate it in %s seconds. Results will be saved in %s' % (str(timeout), str(output))); sys.stdout.flush();
        print('\nCountdown:'); sys.stdout.flush();
        for i in range(0,5):
                print(str(5-i))
                sys.stdout.flush()
                time.sleep(1)
        print('0\nStarting to profile...'); sys.stdout.flush();
        pthread.start() # starting the thread
        time.sleep(float(timeout)) # after this amount of seconds, the thread gets killed and the profiler will end its job
        print('\n\nFinishing the profile and saving to the file %s' % str(output)); sys.stdout.flush();
        pthread.kill() # we must end the main function in order for the profiler to output its results (if we didn't launch a thread and just closed the process, it would have done no result)
    # Run in full length mode (we run the function until it ends)
    else:
        print("Running the profiler, please wait until the process terminates by itself (if you forcequit before, the profile won't be saved)")
        _profile()
    print('=> Functions Profile done !')
    return True
1185941
def _run_cromwell(args):
    
    main_file, json_file, project_name = _get_main_and_json(args.directory)
    work_dir = utils.safe_makedir(os.path.join(os.getcwd(), "cromwell_work"))
    final_dir = utils.safe_makedir(os.path.join(work_dir, "final"))
    if args.no_container:
        _remove_bcbiovm_path()
    log_file = os.path.join(work_dir, "%s-cromwell.log" % project_name)
    metadata_file = os.path.join(work_dir, "%s-metadata.json" % project_name)
    option_file = os.path.join(work_dir, "%s-options.json" % project_name)
    cromwell_opts = {"final_workflow_outputs_dir": final_dir,
                     "default_runtime_attributes": {"bootDiskSizeGb": 20}}
    with open(option_file, "w") as out_handle:
        json.dump(cromwell_opts, out_handle)

    cmd = ["cromwell", "-Xms1g", "-Xmx%s" % _estimate_runner_memory(json_file),
           "run", "--type", "CWL",
           "-Dconfig.file=%s" % hpc.create_cromwell_config(args, work_dir, json_file)]
    cmd += hpc.args_to_cromwell_cl(args)
    cmd += ["--metadata-output", metadata_file, "--options", option_file,
            "--inputs", json_file, main_file]
    with utils.chdir(work_dir):
        _run_tool(cmd, not args.no_container, work_dir, log_file)
        if metadata_file and utils.file_exists(metadata_file):
            with open(metadata_file) as in_handle:
                metadata = json.load(in_handle)
            if metadata["status"] == "Failed":
                _cromwell_debug(metadata)
                sys.exit(1)
            else:
                _cromwell_move_outputs(metadata, final_dir)
217095
def write_resultfiles(self):
        
        t0=time.time()
        writer = ow.output_writer(output_dictionary=self.result)
        if len(self.options.outpath)>=3 and self.options.outpath[-3:]==".h5":
            writer.write_hdf5(filename=self.options.outpath,timestamp=self.options.timestamp)
        else:
            writer.write_txt(outdir=self.options.outpath,timestamp=self.options.timestamp,delimiter=self.options.delimiter,float_format=self.options.float_format)
        t1=time.time()
        print(("Elapsed time for writing the output files is %.2f seconds" % (t1-t0)))
190176
def profile(model_specification, results_directory, process):
    
    model_specification = Path(model_specification)
    results_directory = Path(results_directory)

    out_stats_file = results_directory / f'{model_specification.name}'.replace('yaml', 'stats')
    command = f'run_simulation("{model_specification}", "{results_directory}")'
    cProfile.runctx(command, globals=globals(), locals=locals(), filename=out_stats_file)

    if process:
        out_txt_file = results_directory / (out_stats_file.name + '.txt')
        with open(out_txt_file, 'w') as f:
            p = pstats.Stats(str(out_stats_file), stream=f)
            p.sort_stats('cumulative')
            p.print_stats()
273677
def output_folder(self, value):
        
        self._output_folder = value
        if not os.path.exists(self._output_folder):
            os.makedirs(self._output_folder)
628499
def package(self, output=None):
        
        # Make sure we're in a venv.
        self.check_venv()

        # force not to delete the local zip
        self.override_stage_config_setting('delete_local_zip', False)
        # Execute the prebuild script
        if self.prebuild_script:
            self.execute_prebuild_script()
        # Create the Lambda Zip
        self.create_package(output)
        self.callback('zip')
        size = human_size(os.path.getsize(self.zip_path))
        click.echo(click.style("Package created", fg="green", bold=True) + ": " + click.style(self.zip_path, bold=True) + " (" + size + ")")
299317
def init_stats(output_dir, config):
    

    try:
        os.makedirs(output_dir, 0o755)
    except OSError as e:
        print("ERROR: Can not create output directory: %s\n" % e)
        raise

    db_uri = get_db_uri(config, output_dir)

    set_database(db_uri, db, config)

    tables_to_create = [t for t in [Result, Turret] if not t.table_exists()]

    db.get_conn()
    db.create_tables(tables_to_create)
179377
def makeOuputDir(outputDir, force):
    
    if outputDir:
        if exists(outputDir):
            if not force:
                print('Will not overwrite pre-existing files. Use --force to '
                      'make me.', file=sys.stderr)
                sys.exit(1)
        else:
            mkdir(outputDir)
    else:
        outputDir = mkdtemp()
        print('Writing output files to %s' % outputDir)

    return outputDir
322597
def parse_config_input_output(args=sys.argv):
    
    parser = argparse.ArgumentParser(
        description='Process the input files using the given config')
    parser.add_argument(
        'config_file',
        help='Configuration file.',
        metavar='FILE', type=extant_file)
    parser.add_argument(
        'input_dir',
        help='Directory containing the input files.',
        metavar='DIR', type=extant_dir)
    parser.add_argument(
        'output_dir',
        help='Directory where the output files should be saved.',
        metavar='DIR', type=extant_dir)
    return parser.parse_args(args[1:])