Compute a motor command to reach the sensory goal s_g. It is a shortcut for self.infer(self.conf.s_dims, self.conf.m_dims, s_g)
426018
def inverse_prediction(self, s_g):
        
        return self.infer(self.conf.s_dims, self.conf.m_dims, s_g)
1075060
def step(self):
        """"""
        op = self.code[self.instruction_pointer]
        self.instruction_pointer += 1
        op(self)
979708
def actions_to_reach_reduce(state_idx, target)
      rule = target.heads.map(&:rule).min_by do |r|
        r.symbols.flat_map(&:shortest_production).size
      end

      actions, cur_state = [], state_idx
      rule.symbols.each do |sym|
        if sym.terminal?
          actions << sym
        else
          actions.concat(actions_to_reach_reduce(cur_state, sym))
        end
        cur_state = transition_graph[cur_state][sym]
      end
      cur_state = transition_graph[state_idx][target]

      actions << ReduceStep.new(state_idx, cur_state, rule, target)
    end
968325
def run(self):
        
        x, y = 1,0  # set the direction
        num_steps = 0
        while self.s.get_state() != 'Halted':
            self.s.command({'name':'walk', 'type':'move', 'direction':[x, y]}, self.a1)
            self.s.command({'name':'walk', 'type':'run', 'direction':[x, y+1]}, self.a2)
            num_steps += 1
            if num_steps >= 3:
                break
        for a in self.s.agents:
            print(a.name, 'finished at position ', a.coords['x'], a.coords['y'])
972122
def infer_x(self, y):
        
        assert len(y) == self.fmodel.dim_y, "Wrong dimension for y. Expected %i, got %i" % (self.fmodel.dim_y, len(y))
        self.goal = np.array(y)
1132907
async def stepper_step(self, command):
        
        speed = int(command[0])
        num_steps = int(command[1])
        await self.core.stepper_step(speed, num_steps)
218753
def learn(self, state_key, limit=1000):
        '''
        
        '''
        self.t = 1
        while self.t <= limit:
            next_action_list = self.extract_possible_actions(state_key)
            if len(next_action_list):
                action_key = self.select_action(
                    state_key=state_key,
                    next_action_list=next_action_list
                )
                reward_value = self.observe_reward_value(state_key, action_key)

            if len(next_action_list):
                # Max-Q-Value in next action time.
                next_state_key = self.update_state(
                    state_key=state_key,
                    action_key=action_key
                )

                next_next_action_list = self.extract_possible_actions(next_state_key)
                next_action_key = self.predict_next_action(next_state_key, next_next_action_list)
                next_max_q = self.extract_q_df(next_state_key, next_action_key)

                # Update Q-Value.
                self.update_q(
                    state_key=state_key,
                    action_key=action_key,
                    reward_value=reward_value,
                    next_max_q=next_max_q
                )
                # Update State.
                state_key = next_state_key

            # Normalize.
            self.normalize_q_value()
            self.normalize_r_value()

            # Vis.
            self.visualize_learning_result(state_key)
            # Check.
            if self.check_the_end_flag(state_key) is True:
                break

            # Epsode.
            self.t += 1
1101246
function(num_states, num_actions, opt) {
    var opt = opt || {};
    // in number of time steps, of temporal memory
    // the ACTUAL input to the net will be (x,a) temporal_window times, and followed by current x
    // so to have no information from previous time step going into value function, set to 0.
    this.temporal_window = typeof opt.temporal_window !== 'undefined' ? opt.temporal_window : 1; 
    // size of experience replay memory
    this.experience_size = typeof opt.experience_size !== 'undefined' ? opt.experience_size : 30000;
    // number of examples in experience replay memory before we begin learning
    this.start_learn_threshold = typeof opt.start_learn_threshold !== 'undefined'? opt.start_learn_threshold : Math.floor(Math.min(this.experience_size*0.1, 1000)); 
    // gamma is a crucial parameter that controls how much plan-ahead the agent does. In [0,1]
    this.gamma = typeof opt.gamma !== 'undefined' ? opt.gamma : 0.8;
    
    // number of steps we will learn for
    this.learning_steps_total = typeof opt.learning_steps_total !== 'undefined' ? opt.learning_steps_total : 100000;
    // how many steps of the above to perform only random actions (in the beginning)?
    this.learning_steps_burnin = typeof opt.learning_steps_burnin !== 'undefined' ? opt.learning_steps_burnin : 3000;
    // what epsilon value do we bottom out on? 0.0 => purely deterministic policy at end
    this.epsilon_min = typeof opt.epsilon_min !== 'undefined' ? opt.epsilon_min : 0.05;
    // what epsilon to use at test time? (i.e. when learning is disabled)
    this.epsilon_test_time = typeof opt.epsilon_test_time !== 'undefined' ? opt.epsilon_test_time : 0.01;
    
    // advanced feature. Sometimes a random action should be biased towards some values
    // for example in flappy bird, we may want to choose to not flap more often
    if(typeof opt.random_action_distribution !== 'undefined') {
      // this better sum to 1 by the way, and be of length this.num_actions
      this.random_action_distribution = opt.random_action_distribution;
      if(this.random_action_distribution.length !== num_actions) {
        console.log('TROUBLE. random_action_distribution should be same length as num_actions.');
      }
      var a = this.random_action_distribution;
      var s = 0.0; for(var k=0;k<a.length;k++) { s+= a[k]; }
      if(Math.abs(s-1.0)>0.0001) { console.log('TROUBLE. random_action_distribution should sum to 1!'); }
    } else {
      this.random_action_distribution = [];
    }
    
    // states that go into neural net to predict optimal action look as
    // x0,a0,x1,a1,x2,a2,...xt
    // this variable controls the size of that temporal window. Actions are
    // encoded as 1-of-k hot vectors
    this.net_inputs = num_states * this.temporal_window + num_actions * this.temporal_window + num_states;
    this.num_states = num_states;
    this.num_actions = num_actions;
    this.window_size = Math.max(this.temporal_window, 2); // must be at least 2, but if we want more context even more
    this.state_window = new Array(this.window_size);
    this.action_window = new Array(this.window_size);
    this.reward_window = new Array(this.window_size);
    this.net_window = new Array(this.window_size);
    
    // create [state -> value of all possible actions] modeling net for the value function
    var layer_defs = [];
    if(typeof opt.layer_defs !== 'undefined') {
      // this is an advanced usage feature, because size of the input to the network, and number of
      // actions must check out. This is not very pretty Object Oriented programming but I can't see
      // a way out of it :(
      layer_defs = opt.layer_defs;
      if(layer_defs.length < 2) { console.log('TROUBLE! must have at least 2 layers'); }
      if(layer_defs[0].type !== 'input') { console.log('TROUBLE! first layer must be input layer!'); }
      if(layer_defs[layer_defs.length-1].type !== 'regression') { console.log('TROUBLE! last layer must be input regression!'); }
      if(layer_defs[0].out_depth * layer_defs[0].out_sx * layer_defs[0].out_sy !== this.net_inputs) {
        console.log('TROUBLE! Number of inputs must be num_states * temporal_window + num_actions * temporal_window + num_states!');
      }
      if(layer_defs[layer_defs.length-1].num_neurons !== this.num_actions) {
        console.log('TROUBLE! Number of regression neurons should be num_actions!');
      }
    } else {
      // create a very simple neural net by default
      layer_defs.push({type:'input', out_sx:1, out_sy:1, out_depth:this.net_inputs});
      if(typeof opt.hidden_layer_sizes !== 'undefined') {
        // allow user to specify this via the option, for convenience
        var hl = opt.hidden_layer_sizes;
        for(var k=0;k<hl.length;k++) {
          layer_defs.push({type:'fc', num_neurons:hl[k], activation:'relu'}); // relu by default
        }
      }
      layer_defs.push({type:'regression', num_neurons:num_actions}); // value function output
    }
    this.value_net = new convnetjs.Net();
    this.value_net.makeLayers(layer_defs);
    
    // and finally we need a Temporal Difference Learning trainer!
    var tdtrainer_options = {learning_rate:0.01, momentum:0.0, batch_size:64, l2_decay:0.01};
    if(typeof opt.tdtrainer_options !== 'undefined') {
      tdtrainer_options = opt.tdtrainer_options; // allow user to overwrite this
    }
    this.tdtrainer = new convnetjs.SGDTrainer(this.value_net, tdtrainer_options);
    
    // experience replay
    this.experience = [];
    
    // various housekeeping variables
    this.age = 0; // incremented every backward()
    this.forward_passes = 0; // incremented every forward()
    this.epsilon = 1.0; // controls exploration exploitation tradeoff. Should be annealed over time
    this.latest_reward = 0;
    this.last_input_array = [];
    this.average_reward_window = new cnnutil.Window(1000, 10);
    this.average_loss_window = new cnnutil.Window(1000, 10);
    this.learning = true;
  }
547328
def inference(self, limit=1000):
        '''
        
        '''
        route_list = []
        memory_list = []
        state_key = self.__start_point_tuple
        x, y = state_key
        end_x, end_y = self.__end_point_tuple
        for i in range(limit):
            q_df = self.q_df[self.q_df.state_key == state_key]
            if len(memory_list):
                q_df = q_df[~q_df.action_key.isin(memory_list)]
            if q_df.shape[0] > 1:
                q_df = q_df.sort_values(by=["q_value"], ascending=False)
                action_key = q_df.iloc[0, :]["action_key"]
                q_value = q_df.iloc[0, :]["q_value"]
            elif q_df.shape[0] == 1:
                action_key = q_df.action_key.values[0]
                q_value = q_df.q_value.values[0]
            else:
                action_key_list = self.extract_possible_actions(state_key)
                action_key_list = [v for v in action_key_list if v not in memory_list]
                q_value = 0.0
                if len(action_key_list):
                    action_key = random.choice(action_key_list)
                    _q_df = q_df[q_df.action_key == action_key]
                    if _q_df.shape[0]:
                        q_value = _q_df.q_value.values[0]

            state_key = self.update_state(
                state_key=state_key,
                action_key=action_key
            )
            x, y = state_key
            route_list.append((x, y, q_value))
            memory_list.append(state_key)
            if self.check_the_end_flag(state_key) is True:
                break

        return route_list
64136
def step(self, observations, state, argmax_sampling=False):
        
        action_pd_params, value_output, new_state = self(observations, state)
        actions = self.action_head.sample(action_pd_params, argmax_sampling=argmax_sampling)

        # log likelihood of selected action
        logprobs = self.action_head.logprob(actions, action_pd_params)

        return {
            'actions': actions,
            'values': value_output,
            'logprobs': logprobs,
            'state': new_state
        }
1002897
def infer_cm(tpm):
    
    network_size = tpm.shape[-1]
    all_contexts = tuple(all_states(network_size - 1))
    cm = np.empty((network_size, network_size), dtype=int)
    for a, b in np.ndindex(cm.shape):
        cm[a][b] = infer_edge(tpm, a, b, all_contexts)
    return cm
515749
def mic(self, mechanism, purviews=False):
        
        return self.find_mice(Direction.CAUSE, mechanism, purviews=purviews)
1275168
def wheel(self, *args): # [PSRML], direction, steps
        
        if len(args) == 2:
            PSRML = None
            direction = int(args[0])
            steps = int(args[1])
            stepDelay = None
        elif len(args) == 3:
            PSRML = args[0]
            direction = int(args[1])
            steps = int(args[2])
            stepDelay = None
        elif len(args) == 4:
            PSRML = args[0]
            direction = int(args[1])
            steps = int(args[2])
            stepDelay = int(args[3])
        
        if PSRML is not None:
            self.mouseMove(PSRML)
        Mouse.wheel(direction, steps)
86438
def motor_primitive(self, m):
        
        return bounds_min_max(m, self.conf.m_mins, self.conf.m_maxs)
1079793
def sense(self):
        
        cmd_name = random.choice(self.senses)
        command = getattr(self, cmd_name)
        self.state['last_command'] = cmd_name
        command()
750085
def command(self):
        
        msg = self.gui.status_message
        n = len(msg)
        n_cur = len(self.cursor)
        return msg[:n - n_cur]
1350844
def walk_commands(self):
        """"""
        from .core import GroupMixin
        for command in self.__cog_commands__:
            if command.parent is None:
                yield command
                if isinstance(command, GroupMixin):
                    yield from command.walk_commands()