main loop logic for trial keeper
541568
def main_loop(args):
    ''''''
    
    if not os.path.exists(LOG_DIR):
        os.makedirs(LOG_DIR)
    
    stdout_file = open(STDOUT_FULL_PATH, 'a+')
    stderr_file = open(STDERR_FULL_PATH, 'a+')
    trial_keeper_syslogger = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'trial_keeper', StdOutputType.Stdout, args.log_collection)
    # redirect trial keeper's stdout and stderr to syslog
    trial_syslogger_stdout = RemoteLogger(args.nnimanager_ip, args.nnimanager_port, 'trial', StdOutputType.Stdout, args.log_collection)
    sys.stdout = sys.stderr = trial_keeper_syslogger
    # backward compatibility
    hdfs_host = None
    hdfs_output_dir = None
    if args.hdfs_host:
        hdfs_host = args.hdfs_host
    elif args.pai_hdfs_host:
        hdfs_host = args.pai_hdfs_host
    if args.hdfs_output_dir:
        hdfs_output_dir = args.hdfs_output_dir
    elif args.pai_hdfs_output_dir:
        hdfs_output_dir = args.pai_hdfs_output_dir

    if hdfs_host is not None and args.nni_hdfs_exp_dir is not None:
        try:
            if args.webhdfs_path:
                hdfs_client = HdfsClient(hosts='{0}:80'.format(hdfs_host), user_name=args.pai_user_name, webhdfs_path=args.webhdfs_path, timeout=5)
            else:
                # backward compatibility
                hdfs_client = HdfsClient(hosts='{0}:{1}'.format(hdfs_host, '50070'), user_name=args.pai_user_name, timeout=5)
        except Exception as e:
            nni_log(LogType.Error, 'Create HDFS client error: ' + str(e))
            raise e
        copyHdfsDirectoryToLocal(args.nni_hdfs_exp_dir, os.getcwd(), hdfs_client)

    # Notice: We don't appoint env, which means subprocess wil inherit current environment and that is expected behavior
    log_pipe_stdout = trial_syslogger_stdout.get_pipelog_reader()
    process = Popen(args.trial_command, shell = True, stdout = log_pipe_stdout, stderr = log_pipe_stdout)
    nni_log(LogType.Info, 'Trial keeper spawns a subprocess (pid {0}) to run command: {1}'.format(process.pid, shlex.split(args.trial_command)))

    while True:
        retCode = process.poll()
        # child worker process exits and all stdout data is read
        if retCode is not None and log_pipe_stdout.set_process_exit() and log_pipe_stdout.is_read_completed == True:
            nni_log(LogType.Info, 'subprocess terminated. Exit code is {}. Quit'.format(retCode))
            if hdfs_output_dir is not None:
                # Copy local directory to hdfs for OpenPAI
                nni_local_output_dir = os.environ['NNI_OUTPUT_DIR']
                try:
                    if copyDirectoryToHdfs(nni_local_output_dir, hdfs_output_dir, hdfs_client):
                        nni_log(LogType.Info, 'copy directory from {0} to {1} success!'.format(nni_local_output_dir, hdfs_output_dir))
                    else:
                        nni_log(LogType.Info, 'copy directory from {0} to {1} failed!'.format(nni_local_output_dir, hdfs_output_dir))
                except Exception as e:
                    nni_log(LogType.Error, 'HDFS copy directory got exception: ' + str(e))
                    raise e

            ## Exit as the retCode of subprocess(trial)
            exit(retCode)
            break

        time.sleep(2)
738127
function processRow(row) {
        if (!row) row = {};
        for (var i = 0; i < hooks.length; i++) {
            if (hooks[i].call(row, req, row) === true) return false;
        }
        return true;
    }
87260
function retry(isDone, next) {
            var currentTrial = 0,
                maxRetry = 50,
                interval = 10,
                isTimeout = false;
            var id = window.setInterval(
                function() {
                    if (isDone()) {
                        window.clearInterval(id);
                        next(isTimeout);
                    }
                    if (currentTrial++ > maxRetry) {
                        window.clearInterval(id);
                        isTimeout = true;
                        next(isTimeout);
                    }
                },
                10
            );
        }
1167863
def export_trials_data(args):
    
    nni_config = Config(get_config_filename(args))
    rest_port = nni_config.get_config('restServerPort')
    rest_pid = nni_config.get_config('restServerPid')
    if not detect_process(rest_pid):
        print_error('Experiment is not running...')
        return
    running, response = check_rest_server_quick(rest_port)
    if running:
        response = rest_get(trial_jobs_url(rest_port), 20)
        if response is not None and check_response(response):
            content = json.loads(response.text)
            # dframe = pd.DataFrame.from_records([parse_trial_data(t_data) for t_data in content])
            # dframe.to_csv(args.csv_path, sep='\t')
            records = parse_trial_data(content)
            if args.type == 'json':
                json_records = []
                for trial in records:
                    value = trial.pop('reward', None)
                    trial_id =  trial.pop('id', None)
                    json_records.append({'parameter': trial, 'value': value, 'id': trial_id})
            with open(args.path, 'w') as file:
                if args.type == 'csv':
                    writer = csv.DictWriter(file, set.union(*[set(r.keys()) for r in records]))
                    writer.writeheader()
                    writer.writerows(records)
                else:
                    json.dump(json_records, file)
        else:
            print_error('Export failed...')
    else:
        print_error('Restful server is not Running')
777996
def add_trial(trial)
      RRRSpec.redis.rpush(RRRSpec.make_key(key, 'trial'),
                          trial.key)
    end
1189245
def main():
    
    startLogging(stdout)
    checker = InMemoryUsernamePasswordDatabaseDontUse()
    checker.addUser("testuser", "examplepass")
    realm = AdditionRealm()
    factory = CredAMPServerFactory(Portal(realm, [checker]))
    reactor.listenTCP(7805, factory)
    reactor.run()
1106220
def cleanup_full(self, trial_runner):
        """"""
        for trial in self.current_trials():
            if (trial.status == Trial.PAUSED):
                trial_runner.stop_trial(trial)
1065487
public function onGenericTrial(): bool
    {
        return $this->trial_ends_at && Carbon::now()->lt(Carbon::createFromFormat('Y-m-d H:i:s', $this->trial_ends_at));
    }
201579
def run_trial(self, trial_id=0):
        
        # Set-up trial environment and graph
        self.env = NetworkEnvironment(self.G.copy(), initial_time=0, **self.environment_params)
        # self.G = self.initial_topology.copy()
        # self.trial_params = deepcopy(self.global_params)

        # Set up agents on nodes
        print('Setting up agents...')
        self.setup_network_agents()

        # Set up environmental agent
        if self.environment_agent_type:
            env_agent = self.environment_agent_type(environment=self.env)

        # Set up logging
        logging_interval = self.logging_interval
        logger = BaseLoggingAgent(environment=self.env, dir_path=self.dir_path, logging_interval=logging_interval)

        # Run trial
        self.env.run(until=self.until)

        # Save output as pickled objects
        logger.save_trial_state_history(trial_id=trial_id)
548664
def _try_recover(self, trial, error_msg):
        
        try:
            self.trial_executor.stop_trial(
                trial,
                error=error_msg is not None,
                error_msg=error_msg,
                stop_logger=False)
            trial.result_logger.flush()
            if self.trial_executor.has_resources(trial.resources):
                logger.info("Attempting to recover"
                            " trial state from last checkpoint.")
                self.trial_executor.start_trial(trial)
                if trial.status == Trial.ERROR:
                    raise RuntimeError("Trial did not start correctly.")
            else:
                logger.debug("Notifying Scheduler and requeueing trial.")
                self._requeue_trial(trial)
        except Exception:
            logger.exception("Error recovering trial from checkpoint, abort.")
            self._scheduler_alg.on_trial_error(self, trial)
            self._search_alg.on_trial_complete(trial.trial_id, error=True)
258403
def main_loop(self, timeout=None):
        
        if self.trigger_event.wait(timeout):
            try:
                self.func()
            except Exception as e:
                self.logger.warning("Got an exception running {func}: {e}".format(func=self.func, e=str(e)))
            finally:
                self.trigger_event.clear()
1256999
static void processFirstRow(BaseRow currentRow, ValueState<Boolean> state, Collector<BaseRow> out)
			throws Exception {
		// Check message should be accumulate
		Preconditions.checkArgument(BaseRowUtil.isAccumulateMsg(currentRow));
		// ignore record with timestamp bigger than preRow
		if (state.value() != null) {
			return;
		}
		state.update(true);
		out.collect(currentRow);
	}
91042
def on_trial_remove(self, trial_runner, trial):
        """"""
        if trial.status is Trial.PAUSED and trial in self._results:
            self._completed_trials.add(trial)
1212293
def rl_loop():
  """"""

  state = State()

  if FLAGS.checkpoint_dir:
    # Start from a partially trained model.
    initialize_from_checkpoint(state)
  else:
    # Play the first round of selfplay games with a fake model that returns
    # random noise. We do this instead of playing multiple games using a single
    # model bootstrapped with random noise to avoid any initial bias.
    wait(selfplay(state, 'bootstrap'))

    # Train a real model from the random selfplay games.
    tf_records = get_golden_chunk_records()
    state.iter_num += 1
    wait(train(state, tf_records))

    # Select the newly trained model as the best.
    state.best_model_name = state.train_model_name
    state.gen_num += 1

    # Run selfplay using the new model.
    wait(selfplay(state))

  # Now start the full training loop.
  while state.iter_num <= FLAGS.iterations:
    # Build holdout glob before incrementing the iteration number because we
    # want to run validation on the previous generation.
    holdout_glob = os.path.join(fsdb.holdout_dir(), '%06d-*' % state.iter_num,
                                '*')

    # Train on shuffled game data from recent selfplay rounds.
    tf_records = get_golden_chunk_records()
    state.iter_num += 1
    wait(train(state, tf_records))

    if FLAGS.parallel_post_train:
      # Run eval, validation & selfplay in parallel.
      model_win_rate, _, _ = wait([
          evaluate_trained_model(state),
          validate(state, holdout_glob),
          selfplay(state)])
    else:
      # Run eval, validation & selfplay sequentially.
      model_win_rate = wait(evaluate_trained_model(state))
      wait(validate(state, holdout_glob))
      wait(selfplay(state))

    if model_win_rate >= FLAGS.gating_win_rate:
      # Promote the trained model to the best model and increment the generation
      # number.
      state.best_model_name = state.train_model_name
      state.gen_num += 1
1121662
def _create_trial_info(self, expr_dir):
        
        meta = self._build_trial_meta(expr_dir)

        self.logger.debug("Create trial for %s" % meta)

        trial_record = TrialRecord.from_json(meta)
        trial_record.save()
1087294
def set_trial_config(experiment_config, port, config_file_name):
    ''''''
    request_data = dict()
    request_data['trial_config'] = experiment_config['trial']
    response = rest_put(cluster_metadata_url(port), json.dumps(request_data), REST_TIME_OUT)
    if check_response(response):
        return True
    else:
        print('Error message is {}'.format(response.text))
        _, stderr_full_path = get_log_path(config_file_name)
        if response:
            with open(stderr_full_path, 'a+') as fout:
                fout.write(json.dumps(json.loads(response.text), indent=4, sort_keys=True, separators=(',', ':')))
        return False
138612
function() {
//YAHOO.example.Performance.trialStart = new Date();

    this._oChainRender.stop();

    this.fireEvent("beforeRenderEvent");

    var i, j, k, len, allRecords;

    var oPaginator = this.get('paginator');
    // Paginator is enabled, show a subset of Records and update Paginator UI
    if(oPaginator) {
        allRecords = this._oRecordSet.getRecords(
                        oPaginator.getStartIndex(),
                        oPaginator.getRowsPerPage());
    }
    // Not paginated, show all records
    else {
        allRecords = this._oRecordSet.getRecords();
    }

    // From the top, update in-place existing rows, so as to reuse DOM elements
    var elTbody = this._elTbody,
        loopN = this.get("renderLoopSize"),
        nRecordsLength = allRecords.length;
    
    // Table has rows
    if(nRecordsLength > 0) {                
        elTbody.style.display = "none";
        while(elTbody.lastChild) {
            elTbody.removeChild(elTbody.lastChild);
        }
        elTbody.style.display = "";

        // Set up the loop Chain to render rows
        this._oChainRender.add({
            method: function(oArg) {
                if((this instanceof DT) && this._sId) {
                    var i = oArg.nCurrentRecord,
                        endRecordIndex = ((oArg.nCurrentRecord+oArg.nLoopLength) > nRecordsLength) ?
                                nRecordsLength : (oArg.nCurrentRecord+oArg.nLoopLength),
                        elRow, nextSibling;

                    elTbody.style.display = "none";
                    
                    for(; i<endRecordIndex; i++) {
                        elRow = Dom.get(allRecords[i].getId());
                        elRow = elRow || this._addTrEl(allRecords[i]);
                        nextSibling = elTbody.childNodes[i] || null;
                        elTbody.insertBefore(elRow, nextSibling);
                    }
                    elTbody.style.display = "";
                    
                    // Set up for the next loop
                    oArg.nCurrentRecord = i;
                }
            },
            scope: this,
            iterations: (loopN > 0) ? Math.ceil(nRecordsLength/loopN) : 1,
            argument: {
                nCurrentRecord: 0,//nRecordsLength-1,  // Start at first Record
                nLoopLength: (loopN > 0) ? loopN : nRecordsLength
            },
            timeout: (loopN > 0) ? 0 : -1
        });
        
        // Post-render tasks
        this._oChainRender.add({
            method: function(oArg) {
                if((this instanceof DT) && this._sId) {
                    while(elTbody.rows.length > nRecordsLength) {
                        elTbody.removeChild(elTbody.lastChild);
                    }
                    this._setFirstRow();
                    this._setLastRow();
                    this._setRowStripes();
                    this._setSelections();
                }
            },
            scope: this,
            timeout: (loopN > 0) ? 0 : -1
        });
     
    }
    // Table has no rows
    else {
        // Set up the loop Chain to delete rows
        var nTotal = elTbody.rows.length;
        if(nTotal > 0) {
            this._oChainRender.add({
                method: function(oArg) {
                    if((this instanceof DT) && this._sId) {
                        var i = oArg.nCurrent,
                            loopN = oArg.nLoopLength,
                            nIterEnd = (i - loopN < 0) ? -1 : i - loopN;
    
                        elTbody.style.display = "none";
                        
                        for(; i>nIterEnd; i--) {
                            elTbody.deleteRow(-1);
                        }
                        elTbody.style.display = "";
                        
                        // Set up for the next loop
                        oArg.nCurrent = i;
                    }
                },
                scope: this,
                iterations: (loopN > 0) ? Math.ceil(nTotal/loopN) : 1,
                argument: {
                    nCurrent: nTotal, 
                    nLoopLength: (loopN > 0) ? loopN : nTotal
                },
                timeout: (loopN > 0) ? 0 : -1
            });
        }
    }
    this._runRenderChain();
}