An agent is in state0 and does action0 environment then assigns reward0 and provides new state, state1 Experience nodes store all this information, which is used in the Q-learning update step
422250
function(state0, action0, reward0, state1) {
    this.state0 = state0;
    this.action0 = action0;
    this.reward0 = reward0;
    this.state1 = state1;
  }
737409
def rollout(self, batch_info: BatchInfo, model: RlModel, number_of_steps: int) -> Rollout:
        
        assert not model.is_recurrent, "Replay env roller does not support recurrent models"

        accumulator = TensorAccumulator()
        episode_information = []  # List of dictionaries with episode information

        for step_idx in range(number_of_steps):
            step = model.step(self.last_observation)

            replay_extra_information = {}

            accumulator.add('observations', self.last_observation_cpu)

            # Add step to the tensor accumulator
            for name, tensor in step.items():
                tensor_cpu = tensor.cpu()
                accumulator.add(name, tensor_cpu)

                if name != 'actions':
                    replay_extra_information[name] = tensor_cpu.numpy()

            actions_numpy = step['actions'].detach().cpu().numpy()
            new_obs, new_rewards, new_dones, new_infos = self.environment.step(actions_numpy)

            # Store rollout in the experience replay buffer
            self.replay_buffer.store_transition(
                frame=self.last_observation_cpu.numpy(),
                action=actions_numpy,
                reward=new_rewards,
                done=new_dones,
                extra_info=replay_extra_information
            )

            # Done is flagged true when the episode has ended AND the frame we see is already a first frame from the
            # next episode

            dones_tensor = torch.from_numpy(new_dones.astype(np.float32)).clone()
            accumulator.add('dones', dones_tensor)

            self.last_observation_cpu = torch.from_numpy(new_obs).clone()
            self.last_observation = self.last_observation_cpu.to(self.device)
            accumulator.add('rewards', torch.from_numpy(new_rewards.astype(np.float32)).clone())

            episode_information.append(new_infos)

        accumulated_tensors = accumulator.result()

        return Trajectories(
            num_steps=accumulated_tensors['observations'].size(0),
            num_envs=accumulated_tensors['observations'].size(1),
            environment_information=episode_information,
            transition_tensors=accumulated_tensors,
            rollout_tensors={
                'final_values': model.value(self.last_observation).cpu()
            }
        )
283520
def _on_state(self, state, client):
        
        def cb(outputs):
            try:
                distrib, value = outputs.result()
            except CancelledError:
                logger.info("Client {} cancelled.".format(client.ident))
                return
            assert np.all(np.isfinite(distrib)), distrib
            action = np.random.choice(len(distrib), p=distrib)
            client.memory.append(TransitionExperience(
                state, action, reward=None, value=value, prob=distrib[action]))
            self.send_queue.put([client.ident, dumps(action)])
        self.async_predictor.put_task([state], cb)
438099
def forward(self, agent_qs, states):
        
        bs = agent_qs.size(0)
        states = states.reshape(-1, self.state_dim)
        agent_qs = agent_qs.view(-1, 1, self.n_agents)
        # First layer
        w1 = th.abs(self.hyper_w_1(states))
        b1 = self.hyper_b_1(states)
        w1 = w1.view(-1, self.n_agents, self.embed_dim)
        b1 = b1.view(-1, 1, self.embed_dim)
        hidden = F.elu(th.bmm(agent_qs, w1) + b1)
        # Second layer
        w_final = th.abs(self.hyper_w_final(states))
        w_final = w_final.view(-1, self.embed_dim, 1)
        # State-dependent bias
        v = self.V(states).view(-1, 1, 1)
        # Compute final output
        y = th.bmm(hidden, w_final) + v
        # Reshape and return
        q_tot = y.view(bs, -1, 1)
        return q_tot
344944
def multi_agent_example():
    """"""
    env = holodeck.make("UrbanCity")

    cmd0 = np.array([0, 0, -2, 10])
    cmd1 = np.array([0, 0, 5, 10])
    for i in range(10):
        env.reset()
        # This will queue up a new agent to spawn into the environment, given that the coordinates are not blocked.
        sensors = [Sensors.PIXEL_CAMERA, Sensors.LOCATION_SENSOR, Sensors.VELOCITY_SENSOR]
        agent = AgentDefinition("uav1", agents.UavAgent, sensors)
        env.spawn_agent(agent, [1, 1, 5])

        env.set_control_scheme("uav0", ControlSchemes.UAV_ROLL_PITCH_YAW_RATE_ALT)
        env.set_control_scheme("uav1", ControlSchemes.UAV_ROLL_PITCH_YAW_RATE_ALT)

        env.tick()  # Tick the environment once so the second agent spawns before we try to interact with it.

        env.act("uav0", cmd0)
        env.act("uav1", cmd1)
        for _ in range(1000):
            states = env.tick()
            uav0_terminal = states["uav0"][Sensors.TERMINAL]
            uav1_reward = states["uav1"][Sensors.REWARD]
319982
def learn(self, initial_state_key, limit=1000, game_n=1):
        '''
        
            
        '''
        end_flag_list = [False] * len(self.q_learning_list)
        for game in range(game_n):
            state_key = copy.copy(initial_state_key)
            self.t = 1
            while self.t <= limit:
                for i in range(len(self.q_learning_list)):
                    if game + 1 == game_n:
                        self.state_key_list.append((i, copy.copy(state_key)))
                    self.q_learning_list[i].t = self.t
                    next_action_list = self.q_learning_list[i].extract_possible_actions(state_key)

                    if len(next_action_list):
                        action_key = self.q_learning_list[i].select_action(
                            state_key=state_key,
                            next_action_list=next_action_list
                        )
                        reward_value = self.q_learning_list[i].observe_reward_value(state_key, action_key)

                        # Check.
                        if self.q_learning_list[i].check_the_end_flag(state_key) is True:
                            end_flag_list[i] = True

                        # Max-Q-Value in next action time.
                        next_state_key = self.q_learning_list[i].update_state(
                            state_key=state_key,
                            action_key=action_key
                        )
                        next_next_action_list = self.q_learning_list[i].extract_possible_actions(next_state_key)
                        if len(next_next_action_list):
                            next_action_key = self.q_learning_list[i].predict_next_action(
                                next_state_key,
                                next_next_action_list
                            )
                            next_max_q = self.q_learning_list[i].extract_q_df(next_state_key, next_action_key)

                            # Update Q-Value.
                            self.q_learning_list[i].update_q(
                                state_key=state_key,
                                action_key=action_key,
                                reward_value=reward_value,
                                next_max_q=next_max_q
                            )

                        # Update State.
                        state_key = next_state_key

                    # Epsode.
                    self.t += 1
                    self.q_learning_list[i].t = self.t
                    if False not in end_flag_list:
                        break
215797
def update_visited(self):
        
        assert isinstance(self.player.cshape.center, eu.Vector2)
        pos = self.player.cshape.center

        # Helper function
        def set_visited(layer, cell):
            if cell and not cell.properties.get('visited') and cell.tile and cell.tile.id > 0:
                cell.properties['visited'] = True

                self.reward_explore()

                # TODO: Decouple into view rendering
                # Change colour of visited cells
                key = layer.get_key_at_pixel(cell.x, cell.y)
                #layer.set_cell_color(key[0], key[1], [155,155,155])
                layer.set_cell_opacity(key[0], key[1], 255*0.8)
        # End Helper

        # Get the current tile under player
        current = self.visit_layer.get_at_pixel(pos.x, pos.y)

        if current:
            # In spawn square
            if current == self.visit_layer.get_at_pixel(self.spawn.x, self.spawn.y):
                self.reward_goal()

            # Only record/reward exploration when battery is above 50%
            #if self.player.stats['battery'] > 50:
            set_visited(self.visit_layer, current)
            neighbours = self.visit_layer.get_neighbors(current)
            for cell in neighbours:
                neighbour = neighbours[cell]
                set_visited(self.visit_layer, neighbour)
906873
def iteration(self, node_status=True):
        

        # One iteration changes the opinion of at most q voters using the following procedure:
        # - select randomly q voters
        # - compute majority opinion
        # - if tie all agents take opinion +1
        # - if not tie, all agents take majority opinion

        self.clean_initial_status(self.available_statuses.values())

        if self.actual_iteration == 0:
            self.actual_iteration += 1
            delta, node_count, status_delta = self.status_delta(self.status)
            if node_status:
                return {"iteration": 0, "status": self.status.copy(),
                        "node_count": node_count.copy(), "status_delta": status_delta.copy()}
            else:
                return {"iteration": 0, "status": {},
                        "node_count": node_count.copy(), "status_delta": status_delta.copy()}

        # select q random nodes
        discussion_group = [list(self.graph.nodes())[i]
                            for i in np.random.randint(0, self.graph.number_of_nodes(), self.params['model']['q'])]

        # compute majority
        majority_vote = 1
        vote_sum = sum([self.status[node] for node in discussion_group])
        if vote_sum < (self.params["model"]["q"] / 2.0):
            majority_vote = 0  # in case of tie, majority_vote remains 1

        # update status of nodes in discussion group
        delta = {}
        status_delta = {st: 0 for st in self.available_statuses.values()}

        for listener in discussion_group:
            if majority_vote != self.status[listener]:
                delta[listener] = majority_vote

                status_delta[self.status[listener]] += 1
                for x in self.available_statuses.values():
                    if x != self.status[listener]:
                        status_delta[x] -= 1

            self.status[listener] = majority_vote

        # fix
        node_count = {st: len([n for n in self.status if self.status[n] == st])
                      for st in self.available_statuses.values()}

        self.actual_iteration += 1

        if node_status:
            return {"iteration": self.actual_iteration - 1, "status": delta.copy(),
                    "node_count": node_count.copy(), "status_delta": status_delta.copy()}
        else:
            return {"iteration": self.actual_iteration - 1, "status": {},
                    "node_count": node_count.copy(), "status_delta": status_delta.copy()}
531185
def updateQTable( self, reward, current_state ):
        """"""
        
        # retrieve the old action value from the Q-table (indexed by the previous state and the previous action)
        old_q = self.q_table[self.prev_s][self.prev_a]
        
        # TODO: what should the new action value be?
        new_q = old_q
        
        # assign the new action value to the Q-table
        self.q_table[self.prev_s][self.prev_a] = new_q
1273873
def calculate_gradient(self, batch_info, device, model, rollout):
        
        rollout = rollout.to_transitions()

        dones = rollout.batch_tensor('dones')
        rewards = rollout.batch_tensor('rewards')
        observations_next = rollout.batch_tensor('observations_next')
        actions = rollout.batch_tensor('actions')
        observations = rollout.batch_tensor('observations')

        # Calculate value loss - or critic loss
        with torch.no_grad():
            target_next_value = self.target_model.value(observations_next)
            target_value = rewards + (1.0 - dones) * self.discount_factor * target_next_value

        # Value estimation error vs the target network
        model_value = model.value(observations, actions)
        value_loss = F.mse_loss(model_value, target_value)

        # It may seem a bit tricky what I'm doing here, but the underlying idea is simple
        # All other implementations I found keep two separate optimizers for actor and critic
        # and update them separately
        # What I'm trying to do is to optimize them both with a single optimizer
        # but I need to make sure gradients flow correctly
        # From critic loss to critic network only and from actor loss to actor network only

        # Backpropagate value loss to critic only
        value_loss.backward()

        model_action = model.action(observations)
        model_action_value = model.value(observations, model_action)

        policy_loss = -model_action_value.mean()

        model_action_grad = torch.autograd.grad(policy_loss, model_action)[0]

        # Backpropagate actor loss to actor only
        model_action.backward(gradient=model_action_grad)

        return {
            'policy_loss': policy_loss.item(),
            'value_loss': value_loss.item(),
        }
1283345
private void updateAgentState(AsteriskAgentImpl agent, AgentState newState)
    {
        logger.info("Set state of agent " + agent.getAgentId() + " to " + newState);
        synchronized (agent)
        {
            agent.updateState(newState);
        }
    }
1043763
def agents_me_update(self, data, **kwargs):
        ""
        api_path = "/api/v2/agents/me"
        return self.call(api_path, method="PUT", data=data, **kwargs)
1209070
def initialize(self, training_info, model, environment, device):
        
        self.target_model = self.model_factory.instantiate(action_space=environment.action_space).to(device)
        self.target_model.load_state_dict(model.state_dict())
        self.target_model.eval()

        histogram_info = model.histogram_info()

        self.vmin = histogram_info['vmin']
        self.vmax = histogram_info['vmax']

        self.num_atoms = histogram_info['num_atoms']

        self.support_atoms = histogram_info['support_atoms']
        self.atom_delta = histogram_info['atom_delta']
791190
def reward_explore(self):
        
        if not 'explore' in self.mode:
            return
        mode = self.mode['explore']
        if mode and mode['reward'] and self.__test_cond(mode):
            self.player.stats['reward'] += mode['reward']
            self.player.stats['score'] += mode['reward']

            self.player.game_over = self.player.game_over or mode['terminal']
940630
def editor_multi_agent_example():
    
    agent_definitions = [
        AgentDefinition("uav0", agents.UavAgent, [Sensors.PIXEL_CAMERA, Sensors.LOCATION_SENSOR]),
        AgentDefinition("uav1", agents.UavAgent, [Sensors.LOCATION_SENSOR, Sensors.VELOCITY_SENSOR])
    ]
    env = HolodeckEnvironment(agent_definitions, start_world=False)

    cmd0 = np.array([0, 0, -2, 10])
    cmd1 = np.array([0, 0, 5, 10])

    for i in range(10):
        env.reset()
        env.act("uav0", cmd0)
        env.act("uav1", cmd1)
        for _ in range(1000):
            states = env.tick()

            uav0_terminal = states["uav0"][Sensors.TERMINAL]
            uav1_reward = states["uav1"][Sensors.REWARD]
1325149
def observe_root_state_assignments(self, model, prop_name, info):
        
        if info['old']:
            self.relieve_model(info['old'])
        if info['new']:
            self.observe_model(info['new'])
            self.logger.info("Exchange observed old root_state model with newly assigned one. sm_id: {}"
                             "".format(info['new'].state.parent.state_machine_id))
1192421
function Game(node) {

        this.node = node;

        // This updates are never published.
        this.setStateLevel(constants.stateLevels.UNINITIALIZED, 'S');
        this.setStageLevel(constants.stageLevels.UNINITIALIZED, 'S');

        // ## Properties

        /**
         * ### Game.metadata
         *
         * The game's metadata
         *
         * This object is normally filled-in automatically with data
         * from the file `package.json` inside the game folder.
         *
         * Contains at least the following properties:
         *
         *  - name,
         *  - description,
         *  - version
         */
        this.metadata = {
            name:        'A nodeGame game',
            description: 'No description',
            version:     '0.0.1'
        };

        /**
         * ### Game.settings
         *
         * The game's settings
         *
         * This object is normally filled-in automatically with the settings
         * contained in the game folder: `game/game.settings`,
         * depending also on the chosen treatment.
         */
        this.settings = {};

        /**
         * ### Game.pl | playerList
         *
         * The list of players connected to the game
         *
         * The list may be empty, depending on the server settings.
         *
         * Two players with the same id, or any player with id equal to
         * `node.player.id` is not allowed, and it will throw an error.
         */
        this.playerList = this.pl = new PlayerList({
            log: this.node.log,
            logCtx: this.node,
            name: 'pl_' + this.node.nodename
        });

        this.pl.on('insert', function(p) {
            if (p.id === node.player.id) {
                throw new Error('node.game.pl.on.insert: cannot add player ' +
                                'with id equal to node.player.id.');
            }
        });

        /**
         * ### Game.ml | monitorList
         *
         * The list of monitor clients connected to the game
         *
         * The list may be empty, depending on the server settings
         */
        this.monitorList = this.ml = new PlayerList({
            log: this.node.log,
            logCtx: this.node,
            name: 'ml_' + this.node.nodename
        });

        /**
         * ### Game.memory
         *
         * A storage database for the game
         *
         * In the server logic the content of SET messages are
         * automatically inserted in this object
         *
         * @see NodeGameClient.set
         */
        this.memory = new GameDB({
            log: this.node.log,
            logCtx: this.node,
            shared: { node: this.node }
        });

        /**
         * ### Game.plot
         *
         * The Game plot
         *
         * @see GamePlot
         */
        this.plot = new GamePlot(this.node, new Stager());

// TODO: check if we need this.
//        // Overriding stdout for game plot and stager.
//        this.plot.setDefaultLog(function() {
//            // Must use apply, else will be executed in the wrong context.
//            node.log.apply(node, arguments);
//        });

        /**
         * ### Game.role
         *
         * The "role" currently held in this game (if any)
         *
         * @see Game.gotoStep
         * @see Game.setRole
         * @see processGotoStepOptions
         */
        this.role = null;

        /**
         * ### Game.partner
         *
         * The id or alias of the "partner" in this game (if any)
         *
         * Some games are played in pairs, this variable holds the id
         * of the partner player.
         *
         * @see Game.setPartner
         * @see processGotoStepOptions
         */
        this.partner = null;

        /**
         * ### Game.matcher
         *
         * Handles assigning matching tasks
         *
         * Assigns roles to players, players to players, etc.
         *
         * @see Game.gotoStep
         */
        this.matcher = MatcherManager ? new MatcherManager(this.node) : null;

        /**
         * ### Game.timer
         *
         * Default game timer synced with stager 'timer' property
         *
         * @see GameTimer
         * @see GameTimer.syncWithStager
         */
        this.timer = this.node.timer.createTimer({
            name: 'game_timer',
            stagerSync: true
        });



        // Setting to stage 0.0.0 and starting.
        this.setCurrentGameStage(new GameStage(), 'S');
        this.setStateLevel(constants.stateLevels.STARTING, 'S');

        /**
         * ### Game.paused
         *
         * TRUE, if the game is paused
         *
         * @see Game.pause
         * @see Game.resume
         */
        this.paused = false;

        /**
         * ### Game.pauseCounter
         *
         * Counts the number of times the game was paused
         *
         * @see Game.pause
         * @see Game.resume
         */
        this.pauseCounter = 0;

        /**
         * ### Game.willBeDone
         *
         * TRUE, if DONE was emitted and evaluated successfully
         *
         * If TRUE, when PLAYING is emitted `node.done` is called
         * immediately, and the game tries to step forward.
         *
         * @see NodeGameClient.done
         */
        this.willBeDone = false;

        /**
         * ### Game.globals
         *
         * Object pointing to the current step _globals_ properties
         *
         * Whenever a new step is executed the _globals_ properties of
         * the step are copied here. The _globals_ properties of the previous
         * stage are deleted.
         *
         * @see GamePlot
         * @see Stager
         */
        this.globals = {};

        /**
         * ### Game._steppedSteps
         *
         * Array of steps previously played
         *
         * @see Game.step
         */
        this._steppedSteps = [ new GameStage() ];

        /**
         * ### Game._breakStage
         *
         * Flags to break current stage at next node.done call
         *
         * @see Game.breakStage
         */
        this._breakStage = false;

        /** ### Game.pushManager
         *
         * Handles pushing client to advance to next step
         *
         * @see PushManager
         */
        this.pushManager = new PushManager(this.node);

        /** ### Game.sizeManager
         *
         * Handles changes in the number of connected players
         *
         * @see SizeManager
         */
        this.sizeManager = new SizeManager(this.node);
    }