Create an iteration for the experiment group.
429453
def create_iteration(self, num_suggestions=0):
        """"""
        from db.models.experiment_groups import ExperimentGroupIteration

        search_manager = self.experiment_group.search_manager
        iteration_config = self.experiment_group.iteration_config

        if iteration_config is None:
            iteration = 0
            bracket_iteration = 0
        else:
            should_reschedule = search_manager.should_reschedule(
                iteration=iteration_config.iteration,
                bracket_iteration=iteration_config.bracket_iteration)
            should_reduce_configs = search_manager.should_reduce_configs(
                iteration=iteration_config.iteration,
                bracket_iteration=iteration_config.bracket_iteration)
            if should_reschedule:
                iteration = iteration_config.iteration + 1
                bracket_iteration = 0
            elif should_reduce_configs:
                iteration = iteration_config.iteration
                bracket_iteration = iteration_config.bracket_iteration + 1
            else:
                raise ValueError(
                    'Hyperband create iteration failed for `{}`, '
                    'could not reschedule ot reduce configs'.format(self.experiment_group.id))

        # Create a new iteration config
        iteration_config = HyperbandIterationConfig(
            iteration=iteration,
            num_suggestions=num_suggestions,
            bracket_iteration=bracket_iteration)
        iteration_config.experiment_ids = []
        return ExperimentGroupIteration.objects.create(
            experiment_group=self.experiment_group,
            data=iteration_config.to_dict())
177389
def get_reduced_configs(self):
        """"""
        iteration_config = self.experiment_group.iteration_config
        if iteration_config is None:
            logger.error(
                'Experiment group `%s` attempt to update iteration, but has no iteration',
                self.experiment_group.id,
                extra={'stack': True})
            return
        search_manager = self.experiment_group.search_manager

        # Get the number of experiments to keep
        n_configs_to_keep = search_manager.get_n_config_to_keep_for_iteration(
            iteration=iteration_config.iteration,
            bracket_iteration=iteration_config.bracket_iteration)

        # Get the last group's experiments metrics
        experiments_metrics = self.experiment_group.iteration_config.experiments_metrics
        if experiments_metrics is None:
            raise ExperimentGroupException()

        # Order the experiments
        reverse = Optimization.maximize(
            self.experiment_group.hptuning_config.hyperband.metric.optimization)
        experiments_metrics = sorted(experiments_metrics, key=lambda x: x[1], reverse=reverse)

        # Keep n experiments
        return [xp[0] for xp in experiments_metrics[:n_configs_to_keep]]
567622
def runExperiment( self, e ):
        """"""

        # create the parameter space
        ps = self.parameterSpace()

        # run the experiment at each point
        nb = self.notebook()
        for p in ps:
            #print "Running {p}".format(p = p)
            res = e.set(p).run()
            nb.addResult(res)

        # commit the results
        nb.commit()
1084368
def iterate(self, params, repetition, iteration):
    
    print("\nStarting iteration",iteration)
    print("Learning rate:", self.learningRate if self.lr_scheduler is None
                                              else self.lr_scheduler.get_lr())
    t1 = time.time()
    ret = {}

    # Update dataset epoch when using pre-processed speech dataset
    if self.use_preprocessed_dataset:
      t2 = time.time()
      self.train_loader.dataset.next_epoch()
      self.validation_loader.dataset.next_epoch()
      self.test_loader.dataset.next_epoch()
      self.bg_noise_loader.dataset.next_epoch()
      print("Dataset Load time = {0:.3f} secs, ".format(time.time() - t2))

    # Update learning rate using learning rate scheduler if configured
    if self.lr_scheduler is not None:
      # ReduceLROnPlateau lr_scheduler step should be called after validation,
      # all other lr_schedulers should be called before training
      if params["lr_scheduler"] != "ReduceLROnPlateau":
        self.lr_scheduler.step()

    self.train(params, epoch=iteration, repetition=repetition)

    # Run validation test
    if self.validation_loader is not None:
      validation = self.test(params, self.validation_loader)

      # ReduceLROnPlateau step should be called after validation
      if params["lr_scheduler"] == "ReduceLROnPlateau":
        self.lr_scheduler.step(validation["test_loss"])

      ret["validation"] = validation
      print("Validation: error=", validation["testerror"],
            "entropy=", validation["entropy"],
            "loss=", validation["test_loss"])
      ret.update({"validationerror": validation["testerror"]})

    # Run test set
    if self.test_loader is not None:
      testResults = self.test(params, self.test_loader)
      ret["testResults"] = testResults
      print("Test: error=", testResults["testerror"],
            "entropy=", testResults["entropy"],
            "loss=", testResults["test_loss"])
      ret.update({"testerror": testResults["testerror"]})

    # Run bg noise set
    if self.bg_noise_loader is not None:
      bgResults = self.test(params, self.bg_noise_loader)
      ret["bgResults"] = bgResults
      print("BG noise error=", bgResults["testerror"])
      ret.update({"bgerror": bgResults["testerror"]})

    ret.update({"elapsedTime": time.time() - self.startTime})
    ret.update({"learningRate": self.learningRate if self.lr_scheduler is None
                                                  else self.lr_scheduler.get_lr()})

    # Run noise set
    if params.get("run_noise_tests", False):
      ret.update(self.runNoiseTests(params))
      print("Noise test results: totalCorrect=", ret["totalCorrect"],
            "Test error=", ret["testerror"], ", entropy=", ret["entropy"])

    ret.update({"elapsedTime": time.time() - self.startTime})
    ret.update({"learningRate": self.learningRate if self.lr_scheduler is None
                                                  else self.lr_scheduler.get_lr()})

    print("Iteration time= {0:.3f} secs, "
          "total elapsed time= {1:.3f} mins".format(
            time.time() - t1,ret["elapsedTime"]/60.0))

    return ret
775678
def launch():
    """"""
    try:
        exp = Experiment(db.init_db(drop_all=False))
    except Exception as ex:
        return error_response(
            error_text="Failed to load experiment in /launch: {}".format(str(ex)),
            status=500,
            simple=True,
        )
    try:
        exp.log("Launching experiment...", "-----")
    except IOError as ex:
        return error_response(
            error_text="IOError writing to experiment log: {}".format(str(ex)),
            status=500,
            simple=True,
        )

    try:
        recruitment_details = exp.recruiter.open_recruitment(
            n=exp.initial_recruitment_size
        )
        session.commit()
    except Exception as e:
        return error_response(
            error_text="Failed to open recruitment, check experiment server log "
            "for details: {}".format(str(e)),
            status=500,
            simple=True,
        )

    for task in exp.background_tasks:
        try:
            gevent.spawn(task)
        except Exception:
            return error_response(
                error_text="Failed to spawn task on launch: {}, ".format(task)
                + "check experiment server log for details",
                status=500,
                simple=True,
            )

    if _config().get("replay", False):
        try:
            task = ReplayBackend(exp)
            gevent.spawn(task)
        except Exception:
            return error_response(
                error_text="Failed to launch replay task for experiment."
                "check experiment server log for details",
                status=500,
                simple=True,
            )

    # If the experiment defines a channel, subscribe the experiment to the
    # redis communication channel:
    if exp.channel is not None:
        try:
            from dallinger.experiment_server.sockets import chat_backend

            chat_backend.subscribe(exp, exp.channel)
        except Exception:
            return error_response(
                error_text="Failed to subscribe to chat for channel on launch "
                + "{}".format(exp.channel)
                + ", check experiment server log for details",
                status=500,
                simple=True,
            )

    message = "\n".join(
        (
            "Initial recruitment list:\n{}".format(
                "\n".join(recruitment_details["items"])
            ),
            "Additional details:\n{}".format(recruitment_details["message"]),
        )
    )

    return success_response(recruitment_msg=message)
880994
def reduce_configs(self):
        """"""
        experiment_ids = self.get_reduced_configs()
        experiments = self.experiment_group.experiments.filter(id__in=experiment_ids)
        self.create_iteration()
        iteration_config = self.experiment_group.iteration_config
        hptuning_config = self.experiment_group.hptuning_config
        n_resources = self.experiment_group.search_manager.get_resources_for_iteration(
            iteration=iteration_config.iteration)
        resource_value = self.experiment_group.search_manager.get_n_resources(
            n_resources=n_resources, bracket_iteration=iteration_config.bracket_iteration
        )
        resource_name = hptuning_config.hyperband.resource.name
        resource_value = hptuning_config.hyperband.resource.cast_value(resource_value)

        # Check if we need to resume or restart the experiments
        for experiment in experiments:
            declarations = experiment.declarations
            declarations[resource_name] = resource_value
            declarations_spec = {'declarations': declarations}
            specification = experiment.specification.patch(declarations_spec)
            status_message = 'Hyperband iteration: {}, bracket iteration: {}'.format(
                iteration_config.iteration,
                iteration_config.bracket_iteration)

            if hptuning_config.hyperband.resume:
                experiment.resume(
                    declarations=declarations,
                    config=specification.parsed_data,
                    message=status_message)
            else:
                experiment.restart(
                    experiment_group=self.experiment_group,
                    declarations=declarations,
                    config=specification.parsed_data)
1146290
def sample(self, n_to_sample, **kwargs):
        
        n_to_sample = verify_positive(int(n_to_sample))

        n_remaining = self._max_iter - self.t_

        if n_remaining == 0:
            if (not self.replace) and (self._n_items == self._max_iter):
                raise Exception("All items have already been sampled")
            else:
                raise Exception("No more space available to continue sampling. "
                                "Consider re-initialising with a larger value "
                                "of max_iter.")

        if n_to_sample > n_remaining:
            warnings.warn("Space only remains for {} more iteration(s). "
                          "Setting n_to_sample = {}.".format(n_remaining, \
                          n_remaining))
            n_to_sample = n_remaining

        for _ in range(n_to_sample):
            self._iterate(**kwargs)
1210672
def setup(self):
        ''' 
        '''

        self.selection = EXPFACTORY_EXPERIMENTS
        self.ordered = len(EXPFACTORY_EXPERIMENTS) > 0
        self.data_base = EXPFACTORY_DATA
        self.study_id = EXPFACTORY_SUBID
        self.base = EXPFACTORY_BASE
        self.randomize = EXPFACTORY_RANDOMIZE
        self.headless = EXPFACTORY_HEADLESS

        # Generate variables, if they exist
        self.vars = generate_runtime_vars() or None

        available = get_experiments("%s" % self.base)
        self.experiments = get_selection(available, self.selection)
        self.logger.debug(self.experiments)
        self.lookup = make_lookup(self.experiments)
        final = "\n".join(list(self.lookup.keys()))       

        bot.log("Headless mode: %s" % self.headless)
        bot.log("User has selected: %s" % self.selection)
        bot.log("Experiments Available: %s" %"\n".join(available))
        bot.log("Randomize: %s" % self.randomize)
        bot.log("Final Set \n%s" % final)
955328
def create_groups(self, *names, **kwargs):
        """"""
        return tuple(self.create_group(name, **kwargs) for name in names)
880451
def run(self):
        """"""
        while True:
            try:
                self.perform_iteration()
            except Exception:
                traceback.print_exc()
                pass

            time.sleep(ray_constants.REPORTER_UPDATE_INTERVAL_MS / 1000)
482899
def run_all(logdir, verbose=False):
  
  data = prepare_data()
  rng = random.Random(0)

  base_writer = tf.summary.create_file_writer(logdir)
  with base_writer.as_default():
    experiment = hp.Experiment(hparams=HPARAMS, metrics=METRICS)
    experiment_string = experiment.summary_pb().SerializeToString()
    tf.summary.experimental.write_raw_pb(experiment_string, step=0)
    base_writer.flush()
  base_writer.close()

  sessions_per_group = 2
  num_sessions = flags.FLAGS.num_session_groups * sessions_per_group
  session_index = 0  # across all session groups
  for group_index in xrange(flags.FLAGS.num_session_groups):
    hparams = {h: sample_uniform(h.domain, rng) for h in HPARAMS}
    hparams_string = str(hparams)
    group_id = hashlib.sha256(hparams_string.encode("utf-8")).hexdigest()
    for repeat_index in xrange(sessions_per_group):
      session_id = str(session_index)
      session_index += 1
      if verbose:
        print(
            "--- Running training session %d/%d"
            % (session_index, num_sessions)
        )
        print(hparams_string)
        print("--- repeat #: %d" % (repeat_index + 1))
      run(
          data=data,
          base_logdir=logdir,
          session_id=session_id,
          group_id=group_id,
          hparams=hparams,
      )
266125
def samples(self):
        """"""
        names = self.series.dimensions
        for values in zip(*(getattr(self.series, name) for name in names)):
            yield dict(zip(names, values))
17603
def run(self):
    
    experiment = self._context.experiment()
    if experiment is None:
      raise error.HParamsError(
          "Can't find an HParams-plugin experiment data in"
          " the log directory. Note that it takes some time to"
          " scan the log directory; if you just started"
          " Tensorboard it could be that we haven't finished"
          " scanning it yet. Consider trying again in a"
          " few seconds.")
    return experiment
647629
def _iter_grouped(self):
        """"""
        for indices in self._group_indices:
            yield self._obj.isel(**{self._group_dim: indices})
1196490
function( groupObject ) {

        var
            // Scope for the looped object
            loopObjectScope,

            // Create the nodes list
            nodesList = '',

            // The counter starts from the `min`
            counter = PickerConstructor._.trigger( groupObject.min, groupObject )


        // Loop from the `min` to `max`, incrementing by `i`
        for ( ; counter <= PickerConstructor._.trigger( groupObject.max, groupObject, [ counter ] ); counter += groupObject.i ) {

            // Trigger the `item` function within scope of the object
            loopObjectScope = PickerConstructor._.trigger( groupObject.item, groupObject, [ counter ] )

            // Splice the subgroup and create nodes out of the sub nodes
            nodesList += PickerConstructor._.node(
                groupObject.node,
                loopObjectScope[ 0 ],   // the node
                loopObjectScope[ 1 ],   // the classes
                loopObjectScope[ 2 ]    // the attributes
            )
        }

        // Return the list of nodes
        return nodesList
    }
1234294
def create_experiment(
    run_config,
    hparams,
    model_name,
    problem_name,
    data_dir,
    train_steps,
    eval_steps,
    min_eval_frequency=2000,
    eval_throttle_seconds=600,
    schedule="train_and_evaluate",
    export=False,
    decode_hparams=None,
    use_tfdbg=False,
    use_dbgprofile=False,
    eval_early_stopping_steps=None,
    eval_early_stopping_metric=None,
    eval_early_stopping_metric_delta=None,
    eval_early_stopping_metric_minimize=True,
    eval_timeout_mins=240,
    eval_use_test_set=False,
    use_tpu=False,
    use_tpu_estimator=False,
    use_xla=False,
    additional_train_hooks=None,
    additional_eval_hooks=None,
    warm_start_from=None,
    decode_from_file="",
    decode_to_file="",
    decode_reference="",
    std_server_protocol=None):
  """"""
  # HParams
  hparams.add_hparam("model_dir", run_config.model_dir)
  hparams.add_hparam("data_dir", data_dir)
  hparams.add_hparam("train_steps", train_steps)
  hparams.add_hparam("eval_steps", eval_steps)
  hparams.add_hparam("schedule", schedule)
  hparams.add_hparam("warm_start_from", warm_start_from)
  hparams.add_hparam("std_server_protocol", std_server_protocol)
  hparams.add_hparam("eval_freq_in_steps", min_eval_frequency)
  hparams.add_hparam("eval_timeout_mins", eval_timeout_mins)
  if decode_hparams is not None:
    decode_hparams.add_hparam("decode_from_file", decode_from_file)
    if decode_to_file and not decode_hparams.decode_to_file:
      decode_hparams.decode_to_file = decode_to_file
    if decode_reference and not decode_hparams.decode_reference:
      decode_hparams.decode_reference = decode_reference
  add_problem_hparams(hparams, problem_name)

  # Estimator
  estimator = create_estimator(
      model_name,
      hparams,
      run_config,
      schedule=schedule,
      decode_hparams=decode_hparams,
      use_tpu=use_tpu,
      use_tpu_estimator=use_tpu_estimator,
      use_xla=use_xla)

  # Input fns from Problem
  problem = hparams.problem
  train_input_fn = problem.make_estimator_input_fn(tf.estimator.ModeKeys.TRAIN,
                                                   hparams)

  dataset_split = "test" if eval_use_test_set else None
  dataset_kwargs = {"dataset_split": dataset_split}
  eval_input_fn = problem.make_estimator_input_fn(tf.estimator.ModeKeys.EVAL,
                                                  hparams,
                                                  dataset_kwargs=dataset_kwargs)

  # Export
  exporter = None
  if export:
    def compare_fn(best_eval_result, current_eval_result):
      metric = eval_early_stopping_metric or "loss"
      return current_eval_result[metric] < best_eval_result[metric]

    def serving_input_receiver_fn(hparams, decode_hparams, use_tpu):
      return problem.serving_input_fn(hparams, decode_hparams, use_tpu)

    exporter = tf.estimator.BestExporter(
        name="best",
        serving_input_receiver_fn=serving_input_receiver_fn,
        compare_fn=compare_fn,
        assets_extra=problem.export_assets)

  # Hooks
  validation_monitor_kwargs = dict(
      input_fn=eval_input_fn,
      eval_steps=eval_steps,
      every_n_steps=min_eval_frequency,
      early_stopping_rounds=eval_early_stopping_steps,
      early_stopping_metric=eval_early_stopping_metric,
      early_stopping_metric_minimize=eval_early_stopping_metric_minimize)
  dbgprofile_kwargs = {"output_dir": run_config.model_dir}
  early_stopping_kwargs = dict(
      events_dir=os.path.join(run_config.model_dir, "eval_continuous"),
      tag=eval_early_stopping_metric,
      num_plateau_steps=eval_early_stopping_steps,
      plateau_decrease=eval_early_stopping_metric_minimize,
      plateau_delta=eval_early_stopping_metric_delta,
      every_n_steps=min_eval_frequency)

  # Eval on TPU Pods is not supported yet
  if use_tpu and run_config.tpu_config.num_shards > 8 and "eval" in schedule:
    raise ValueError("Eval is not currently supported on a TPU Pod")

  # In-process eval (and possible early stopping)
  if schedule == "continuous_train_and_eval" and min_eval_frequency:
    tf.logging.warn("ValidationMonitor only works with "
                    "--schedule=train_and_evaluate")
  use_validation_monitor = (
      schedule == "train_and_evaluate" and min_eval_frequency)
  # Distributed early stopping
  local_schedules = ["train_and_evaluate", "continuous_train_and_eval"]
  use_early_stopping = (
      schedule not in local_schedules and eval_early_stopping_steps)
  train_hooks, eval_hooks = create_hooks(
      use_tfdbg=use_tfdbg,
      use_dbgprofile=use_dbgprofile,
      dbgprofile_kwargs=dbgprofile_kwargs,
      use_validation_monitor=use_validation_monitor,
      validation_monitor_kwargs=validation_monitor_kwargs,
      use_early_stopping=use_early_stopping,
      early_stopping_kwargs=early_stopping_kwargs)

  hook_context = HookContext(
      estimator=estimator, problem=problem, hparams=hparams)

  train_hooks += t2t_model.T2TModel.get_train_hooks(model_name, hook_context)
  eval_hooks += t2t_model.T2TModel.get_eval_hooks(model_name, hook_context)
  if additional_train_hooks:
    train_hooks += additional_train_hooks
  if additional_eval_hooks:
    eval_hooks += additional_eval_hooks

  train_hooks = tf.contrib.learn.monitors.replace_monitors_with_hooks(
      train_hooks, estimator)
  eval_hooks = tf.contrib.learn.monitors.replace_monitors_with_hooks(
      eval_hooks, estimator)

  train_spec = tf.estimator.TrainSpec(
      train_input_fn, max_steps=train_steps, hooks=train_hooks)
  eval_spec = tf.estimator.EvalSpec(
      eval_input_fn,
      steps=eval_steps,
      hooks=eval_hooks,
      start_delay_secs=0 if hparams.schedule == "evaluate" else 120,
      throttle_secs=eval_throttle_seconds,
      exporters=exporter)

  return T2TExperiment(estimator, hparams, train_spec, eval_spec,
                       use_validation_monitor, decode_hparams)
968830
def run_experiment(self):
        """"""
        try:
            self.sign_up()
            self.participate()
            if self.sign_off():
                self.complete_experiment("worker_complete")
            else:
                self.complete_experiment("worker_failed")
        finally:
            self.driver.quit()