Apply sequence of actions to sequence of environments
        actions -> (observations, rewards, news)

        where 'news' is a boolean vector indicating whether each element is new.
480429
def step(self, action):
        
        obs, rews, news, infos = self.env.step(action)

        self.ret = self.ret * self.gamma + rews

        obs = self._filter_observation(obs)

        if self.ret_rms:
            self.ret_rms.update(np.array([self.ret]))
            rews = np.clip(rews / np.sqrt(self.ret_rms.var + self.epsilon), -self.cliprew, self.cliprew)

        return obs, rews, news, infos
198807
def step(self, observations):
        
        q_values = self(observations)

        return {
            'actions': self.q_head.sample(q_values),
            'q': q_values
        }
177254
function initWithActions (actionType, actions) {
    var prev = actions[0].copy()
      , now
      , i
    for (i=1; i<actions.length; i++) {
        now = actions[i].copy()
        if (now) {
            prev = new actionType({one: prev, two: now})
        } else {
            break
        }
    }

    return prev
}
759666
def _generate_time_steps(self, trajectory_list):
    """"""
    for single_trajectory in trajectory_list:
      assert isinstance(single_trajectory, trajectory.Trajectory)

      # Skip writing trajectories that have only a single time-step -- this
      # could just be a repeated reset.

      if single_trajectory.num_time_steps <= 1:
        continue

      for index, time_step in enumerate(single_trajectory.time_steps):

        # The first time-step doesn't have reward/processed_reward, if so, just
        # setting it to 0.0 / 0 should be OK.
        raw_reward = time_step.raw_reward
        if not raw_reward:
          raw_reward = 0.0

        processed_reward = time_step.processed_reward
        if not processed_reward:
          processed_reward = 0

        action = time_step.action
        if action is None:
          # The last time-step doesn't have action, and this action shouldn't be
          # used, gym's spaces have a `sample` function, so let's just sample an
          # action and use that.
          action = self.action_space.sample()
        action = gym_spaces_utils.gym_space_encode(self.action_space, action)

        if six.PY3:
          # py3 complains that, to_example cannot handle np.int64 !

          action_dtype = self.action_space.dtype
          if action_dtype in [np.int64, np.int32]:
            action = list(map(int, action))
          elif action_dtype in [np.float64, np.float32]:
            action = list(map(float, action))

          # same with processed_reward.
          processed_reward = int(processed_reward)

        assert time_step.observation is not None

        yield {
            TIMESTEP_FIELD: [index],
            ACTION_FIELD:
                action,
            # to_example errors on np.float32
            RAW_REWARD_FIELD: [float(raw_reward)],
            PROCESSED_REWARD_FIELD: [processed_reward],
            # to_example doesn't know bools
            DONE_FIELD: [int(time_step.done)],
            OBSERVATION_FIELD:
                gym_spaces_utils.gym_space_encode(self.observation_space,
                                                  time_step.observation),
        }
1169964
def _process_actions(self, actions, inport, packet):
        '''
        
        '''
        second_stage = []
        for a in actions:
            fn = a(packet=packet, net=self._switchyard_net, controllers=self._controller_connections, inport=inport)
            if (fn):
                second_stage.append(fn)
        for fn in second_stage:
            fn()
648399
def _oneInteraction(self):
        
        self.stepid += 1

        logger.info("Entering simulation period %d." % self.stepid)

        # Apply branches outages.
        if self.branchOutages is not None:
            self.doOutages()

        # Initialise the market.
        self.market.reset()

        # Get an action from each agent and perform it.
        for task, agent in zip(self.tasks, self.agents):
#            if self.do_optimisation[agent]:
#                raise Exception("When using a black-box learning algorithm, "
#                                "only full episodes can be done.")

#            if not task.isFinished():
            observation = task.getObservation()
            agent.integrateObservation(observation)

            action = agent.getAction()
            task.performAction(action)

        # Clear the market.
        self.market.run()

        # Reward each agent appropriately.
        for task, agent in zip(self.tasks, self.agents):
#            if not task.isFinished():
            reward = task.getReward()
            agent.giveReward(reward)

        # Scale loads.
        c = self._pcycle.next()
        for bus in self.market.case.buses:
            bus.p_demand = self.pdemand[bus] * c

        logger.info("")
352064
def train_episode(agent, envs, preprocessors, t_max, render):
    """"""
    num_envs = len(envs)

    # Buffers to hold trajectories, e.g. `env_xs[i]` will hold the observations
    # for environment `i`.
    env_xs, env_as = _2d_list(num_envs), _2d_list(num_envs)
    env_rs, env_vs = _2d_list(num_envs), _2d_list(num_envs)
    episode_rs = np.zeros(num_envs, dtype=np.float)

    for p in preprocessors:
        p.reset()

    observations = [p.preprocess(e.reset())
                    for p, e in zip(preprocessors, envs)]

    done = np.array([False for _ in range(num_envs)])
    all_done = False
    t = 1

    while not all_done:
        if render:
            envs[0].render()

        # NOTE(reed): Reshape to set the data shape.
        agent.model.reshape([('data', (num_envs, preprocessors[0].obs_size))])
        step_xs = np.vstack([o.ravel() for o in observations])

        # Get actions and values for all environments in a single forward pass.
        step_xs_nd = mx.nd.array(step_xs, ctx=agent.ctx)
        data_batch = mx.io.DataBatch(data=[step_xs_nd], label=None)
        agent.model.forward(data_batch, is_train=False)
        _, step_vs, _, step_ps = agent.model.get_outputs()

        step_ps = step_ps.asnumpy()
        step_vs = step_vs.asnumpy()
        step_as = agent.act(step_ps)

        # Step each environment whose episode has not completed.
        for i, env in enumerate(envs):
            if not done[i]:
                obs, r, done[i], _ = env.step(step_as[i])

                # Record the observation, action, value, and reward in the
                # buffers.
                env_xs[i].append(step_xs[i].ravel())
                env_as[i].append(step_as[i])
                env_vs[i].append(step_vs[i][0])
                env_rs[i].append(r)
                episode_rs[i] += r

                # Add 0 as the state value when done.
                if done[i]:
                    env_vs[i].append(0.0)
                else:
                    observations[i] = preprocessors[i].preprocess(obs)

        # Perform an update every `t_max` steps.
        if t == t_max:
            # If the episode has not finished, add current state's value. This
            # will be used to 'bootstrap' the final return (see Algorithm S3
            # in A3C paper).
            step_xs = np.vstack([o.ravel() for o in observations])
            step_xs_nd = mx.nd.array(step_xs, ctx=agent.ctx)
            data_batch = mx.io.DataBatch(data=[step_xs_nd], label=None)
            agent.model.forward(data_batch, is_train=False)
            _, extra_vs, _, _ = agent.model.get_outputs()
            extra_vs = extra_vs.asnumpy()
            for i in range(num_envs):
                if not done[i]:
                    env_vs[i].append(extra_vs[i][0])

            # Perform update and clear buffers.
            env_xs = np.vstack(list(chain.from_iterable(env_xs)))
            agent.train_step(env_xs, env_as, env_rs, env_vs)
            env_xs, env_as = _2d_list(num_envs), _2d_list(num_envs)
            env_rs, env_vs = _2d_list(num_envs), _2d_list(num_envs)
            t = 0

        all_done = np.all(done)
        t += 1

    return episode_rs
64136
def step(self, observations, state, argmax_sampling=False):
        
        action_pd_params, value_output, new_state = self(observations, state)
        actions = self.action_head.sample(action_pd_params, argmax_sampling=argmax_sampling)

        # log likelihood of selected action
        logprobs = self.action_head.logprob(actions, action_pd_params)

        return {
            'actions': actions,
            'values': value_output,
            'logprobs': logprobs,
            'state': new_state
        }
906809
def update(self, mini_batch, num_sequences):
        
        feed_dict = {self.model.batch_size: num_sequences,
                     self.model.sequence_length: self.sequence_length,
                     self.model.mask_input: mini_batch['masks'].flatten(),
                     self.model.returns_holder: mini_batch['discounted_returns'].flatten(),
                     self.model.old_value: mini_batch['value_estimates'].flatten(),
                     self.model.advantage: mini_batch['advantages'].reshape([-1, 1]),
                     self.model.all_old_log_probs: mini_batch['action_probs'].reshape(
                         [-1, sum(self.model.act_size)])}
        if self.use_continuous_act:
            feed_dict[self.model.output_pre] = mini_batch['actions_pre'].reshape(
                [-1, self.model.act_size[0]])
            feed_dict[self.model.epsilon] = mini_batch['random_normal_epsilon'].reshape(
                [-1, self.model.act_size[0]])
        else:
            feed_dict[self.model.action_holder] = mini_batch['actions'].reshape(
                [-1, len(self.model.act_size)])
            if self.use_recurrent:
                feed_dict[self.model.prev_action] = mini_batch['prev_action'].reshape(
                    [-1, len(self.model.act_size)])
            feed_dict[self.model.action_masks] = mini_batch['action_mask'].reshape(
                [-1, sum(self.brain.vector_action_space_size)])
        if self.use_vec_obs:
            feed_dict[self.model.vector_in] = mini_batch['vector_obs'].reshape(
                [-1, self.vec_obs_size])
            if self.use_curiosity:
                feed_dict[self.model.next_vector_in] = mini_batch['next_vector_in'].reshape(
                    [-1, self.vec_obs_size])
        if self.model.vis_obs_size > 0:
            for i, _ in enumerate(self.model.visual_in):
                _obs = mini_batch['visual_obs%d' % i]
                if self.sequence_length > 1 and self.use_recurrent:
                    (_batch, _seq, _w, _h, _c) = _obs.shape
                    feed_dict[self.model.visual_in[i]] = _obs.reshape([-1, _w, _h, _c])
                else:
                    feed_dict[self.model.visual_in[i]] = _obs
            if self.use_curiosity:
                for i, _ in enumerate(self.model.visual_in):
                    _obs = mini_batch['next_visual_obs%d' % i]
                    if self.sequence_length > 1 and self.use_recurrent:
                        (_batch, _seq, _w, _h, _c) = _obs.shape
                        feed_dict[self.model.next_visual_in[i]] = _obs.reshape([-1, _w, _h, _c])
                    else:
                        feed_dict[self.model.next_visual_in[i]] = _obs
        if self.use_recurrent:
            mem_in = mini_batch['memory'][:, 0, :]
            feed_dict[self.model.memory_in] = mem_in
        self.has_updated = True
        run_out = self._execute_model(feed_dict, self.update_dict)
        return run_out
1058138
public function applyBehaviors()
    {
        foreach ($this->behaviors as $behavior) {
            if (!$behavior->isTableModified()) {
                $behavior->getTableModifier()->modifyTable();
                $behavior->setTableModified(true);
            }
        }
    }
1288921
def collect_random_trajectory(env, timesteps=1000):
    

    obs = env.reset()
    dof = env.dof

    for t in range(timesteps):
        action = 0.5 * np.random.randn(dof)
        obs, reward, done, info = env.step(action)
        env.render()
        if t % 100 == 0:
            print(t)
452291
def run_loop(agents, env, max_frames=0, max_episodes=0):
  """"""
  total_frames = 0
  total_episodes = 0
  start_time = time.time()

  observation_spec = env.observation_spec()
  action_spec = env.action_spec()
  for agent, obs_spec, act_spec in zip(agents, observation_spec, action_spec):
    agent.setup(obs_spec, act_spec)

  try:
    while not max_episodes or total_episodes < max_episodes:
      total_episodes += 1
      timesteps = env.reset()
      for a in agents:
        a.reset()
      while True:
        total_frames += 1
        actions = [agent.step(timestep)
                   for agent, timestep in zip(agents, timesteps)]
        if max_frames and total_frames >= max_frames:
          return
        if timesteps[0].last():
          break
        timesteps = env.step(actions)
  except KeyboardInterrupt:
    pass
  finally:
    elapsed_time = time.time() - start_time
    print("Took %.3f seconds for %s steps: %.3f fps" % (
        elapsed_time, total_frames, total_frames / elapsed_time))
54016
def performAction(self, action):
        
        self._lastAction = []

        # Markups chosen for each generator.
        actions = self._allActions[action]

        n = self.numOffbids * len(self.generators)
        markups = actions[:n]
        withholds = actions[n:]

#        print "ALL ACTIONS:", self._allActions
#        print "ACTIONS:", markups, withholds

        self._offbid(markups, withholds)
869087
def play_random(env, steps):
    
    try:
        done = True
        progress = tqdm(range(steps))
        for _ in progress:
            if done:
                _ = env.reset()
            action = env.action_space.sample()
            _, reward, done, info = env.step(action)
            progress.set_postfix(reward=reward, info=info)
            env.render()
    except KeyboardInterrupt:
        pass
    # close the environment
    env.close()
1221242
def _checkRewardsListLike(reward, n_actions, n_states):
    
    try:
        lenR = len(reward)
        if lenR == n_actions:
            dim1, dim2, dim3 = _checkDimensionsListLike(reward)
        elif lenR == n_states:
            dim1 = n_actions
            dim2 = dim3 = lenR
        else:
            raise _error.InvalidError(_MDPERR["R_shape"])
    except AttributeError:
        raise _error.InvalidError(_MDPERR["R_shape"])
    return dim1, dim2, dim3
51248
def setActions( self, actions ):
        
        self._actions = actions
        
        self.uiActionTREE.blockSignals(True)
        self.uiActionTREE.setUpdatesEnabled(False)
        
        self.uiActionTREE.clear()
        actions = actions.items()
        actions.sort(key = lambda x: nativestring(x[1].text()).replace('&', ''))
        
        for key, action in actions:
            item = self.createActionItem(key)
            self.uiActionTREE.addTopLevelItem(item)
        
        self.uiActionTREE.setUpdatesEnabled(True)
        self.uiActionTREE.blockSignals(False)
1244404
def update(self, mini_batch, num_sequences):
        

        feed_dict = {self.model.dropout_rate: self.update_rate,
                     self.model.batch_size: num_sequences,
                     self.model.sequence_length: self.sequence_length}
        if self.use_continuous_act:
            feed_dict[self.model.true_action] = mini_batch['actions']. \
                reshape([-1, self.brain.vector_action_space_size[0]])
        else:
            feed_dict[self.model.true_action] = mini_batch['actions'].reshape(
                [-1, len(self.brain.vector_action_space_size)])
            feed_dict[self.model.action_masks] = np.ones(
                (num_sequences, sum(self.brain.vector_action_space_size)))
        if self.use_vec_obs:
            apparent_obs_size = self.brain.vector_observation_space_size * \
                                self.brain.num_stacked_vector_observations
            feed_dict[self.model.vector_in] = mini_batch['vector_obs'] \
                .reshape([-1,apparent_obs_size])
        for i, _ in enumerate(self.model.visual_in):
            visual_obs = mini_batch['visual_obs%d' % i]
            feed_dict[self.model.visual_in[i]] = visual_obs
        if self.use_recurrent:
            feed_dict[self.model.memory_in] = np.zeros([num_sequences, self.m_size])
        run_out = self._execute_model(feed_dict, self.update_dict)
        return run_out