LSTM seq2seq model with attention, main step used for training.
197230
def lstm_seq2seq_internal_attention(inputs, targets, hparams, train,
                                    inputs_length, targets_length):
  """"""
  with tf.variable_scope("lstm_seq2seq_attention"):
    # Flatten inputs.
    inputs = common_layers.flatten4d3d(inputs)

    # LSTM encoder.
    inputs = tf.reverse_sequence(inputs, inputs_length, seq_axis=1)
    encoder_outputs, final_encoder_state = lstm(
        inputs, inputs_length, hparams, train, "encoder")

    # LSTM decoder with attention.
    shifted_targets = common_layers.shift_right(targets)
    # Add 1 to account for the padding added to the left from shift_right
    targets_length = targets_length + 1
    decoder_outputs = lstm_attention_decoder(
        common_layers.flatten4d3d(shifted_targets), hparams, train, "decoder",
        final_encoder_state, encoder_outputs, inputs_length, targets_length)
    return tf.expand_dims(decoder_outputs, axis=2)
342241
def _create_rnn_architecture(self):
        """"""
        self.inputs = [tf.squeeze(i, [1]) for i in tf.split(
            axis=1, num_or_size_splits=self.num_steps, value=self.inputs)]
        outputs, state = tf.nn.rnn(
            self.cell, self.inputs, initial_state=self._init_state)

        output = tf.reshape(tf.concat(axis=1, values=outputs), [-1, self.num_hidden])
        softmax_w = tf.get_variable(
            "softmax_w", [self.num_hidden, self.vocab_size])
        softmax_b = tf.get_variable("softmax_b", [self.vocab_size])
        logits = tf.add(tf.matmul(output, softmax_w), softmax_b)
        loss = tf.nn.seq2seq.sequence_loss_by_example(
            [logits],
            [tf.reshape(self.input_labels, [-1])],
            [tf.ones([self.batch_size * self.num_steps])])

        self.cost = tf.div(tf.reduce_sum(loss), self.batch_size)
        self.final_state = state
403936
def sym_gen(seq_len):
    
    sentence_shape = train_iter.provide_data[0][1]
    char_sentence_shape = train_iter.provide_data[1][1]
    entities_shape = train_iter.provide_label[0][1]

    X_sent = mx.symbol.Variable(train_iter.provide_data[0].name)
    X_char_sent = mx.symbol.Variable(train_iter.provide_data[1].name)
    Y = mx.sym.Variable(train_iter.provide_label[0].name)

    ###############################
    # Character embedding component
    ###############################
    char_embeddings = mx.sym.Embedding(data=X_char_sent, input_dim=len(char_to_index), output_dim=args.char_embed, name='char_embed')
    char_embeddings = mx.sym.reshape(data=char_embeddings, shape=(0,1,seq_len,-1,args.char_embed), name='char_embed2')

    char_cnn_outputs = []
    for i, filter_size in enumerate(args.char_filter_list):
        # Kernel that slides over entire words resulting in a 1d output
        convi = mx.sym.Convolution(data=char_embeddings, kernel=(1, filter_size, args.char_embed), stride=(1, 1, 1),
                                   num_filter=args.char_filters, name="char_conv_layer_" + str(i))
        acti = mx.sym.Activation(data=convi, act_type='tanh')
        pooli = mx.sym.Pooling(data=acti, pool_type='max', kernel=(1, char_sentence_shape[2] - filter_size + 1, 1),
                               stride=(1, 1, 1), name="char_pool_layer_" + str(i))
        pooli = mx.sym.transpose(mx.sym.Reshape(pooli, shape=(0, 0, 0)), axes=(0, 2, 1), name="cchar_conv_layer_" + str(i))
        char_cnn_outputs.append(pooli)

    # combine features from all filters & apply dropout
    cnn_char_features = mx.sym.Concat(*char_cnn_outputs, dim=2, name="cnn_char_features")
    regularized_cnn_char_features = mx.sym.Dropout(data=cnn_char_features, p=args.dropout, mode='training',
                                                   name='regularized charCnn features')

    ##################################
    # Combine char and word embeddings
    ##################################
    word_embeddings = mx.sym.Embedding(data=X_sent, input_dim=len(word_to_index), output_dim=args.word_embed, name='word_embed')
    rnn_features = mx.sym.Concat(*[word_embeddings, regularized_cnn_char_features], dim=2, name='rnn input')

    ##############################
    # Bidirectional LSTM component
    ##############################

    # unroll the lstm cell in time, merging outputs
    bi_cell.reset()
    output, states = bi_cell.unroll(length=seq_len, inputs=rnn_features, merge_outputs=True)

    # Map to num entity classes
    rnn_output = mx.sym.Reshape(output, shape=(-1, args.lstm_state_size * 2), name='r_output')
    fc = mx.sym.FullyConnected(data=rnn_output, num_hidden=len(entity_to_index), name='fc_layer')

    # reshape back to same shape as loss will be
    reshaped_fc = mx.sym.transpose(mx.sym.reshape(fc, shape=(-1, seq_len, len(entity_to_index))), axes=(0, 2, 1))
    sm = mx.sym.SoftmaxOutput(data=reshaped_fc, label=Y, ignore_label=-1, use_ignore=True, multi_output=True, name='softmax')
    return sm, [v.name for v in train_iter.provide_data], [v.name for v in train_iter.provide_label]
896287
def lstm_bid_encoder(inputs, sequence_length, hparams, train, name):
  """"""

  with tf.variable_scope(name):
    cell_fw = tf.nn.rnn_cell.MultiRNNCell(
        [_dropout_lstm_cell(hparams, train)
         for _ in range(hparams.num_hidden_layers)])

    cell_bw = tf.nn.rnn_cell.MultiRNNCell(
        [_dropout_lstm_cell(hparams, train)
         for _ in range(hparams.num_hidden_layers)])

    ((encoder_fw_outputs, encoder_bw_outputs),
     (encoder_fw_state, encoder_bw_state)) = tf.nn.bidirectional_dynamic_rnn(
         cell_fw,
         cell_bw,
         inputs,
         sequence_length,
         dtype=tf.float32,
         time_major=False)

    encoder_outputs = tf.concat((encoder_fw_outputs, encoder_bw_outputs), 2)
    encoder_states = []

    for i in range(hparams.num_hidden_layers):
      if isinstance(encoder_fw_state[i], tf.nn.rnn_cell.LSTMStateTuple):
        encoder_state_c = tf.concat(
            values=(encoder_fw_state[i].c, encoder_bw_state[i].c),
            axis=1,
            name="encoder_fw_state_c")
        encoder_state_h = tf.concat(
            values=(encoder_fw_state[i].h, encoder_bw_state[i].h),
            axis=1,
            name="encoder_fw_state_h")
        encoder_state = tf.nn.rnn_cell.LSTMStateTuple(
            c=encoder_state_c, h=encoder_state_h)
      elif isinstance(encoder_fw_state[i], tf.Tensor):
        encoder_state = tf.concat(
            values=(encoder_fw_state[i], encoder_bw_state[i]),
            axis=1,
            name="bidirectional_concat")

      encoder_states.append(encoder_state)

    encoder_states = tuple(encoder_states)
    return encoder_outputs, encoder_states
1298642
def _init_evolved_transformer_cache(cache, hparams, batch_size,
                                    attention_init_length, encoder_output,
                                    encoder_decoder_attention_bias,
                                    scope_prefix):
  """"""
  key_channels = hparams.attention_key_channels or hparams.hidden_size
  value_channels = hparams.attention_value_channels or hparams.hidden_size
  num_layers = hparams.num_decoder_layers or hparams.num_hidden_layers
  vars_3d_num_heads = (
      hparams.num_heads if hparams.get("attention_variables_3d") else 0)

  # Add self-attentions.
  if cache is None:
    cache = {}
  cache.update({
      "layer_%d" % layer: {  # pylint: disable=g-complex-comprehension
          _SIXTEEN_HEAD_ATTENTION_NAME: {
              "k":
                  common_attention.split_heads(
                      tf.zeros(
                          [batch_size, attention_init_length, key_channels]),
                      _capped_double_heads(hparams.num_heads)),
              "v":
                  common_attention.split_heads(
                      tf.zeros(
                          [batch_size, attention_init_length, value_channels]),
                      _capped_double_heads(hparams.num_heads)),
          },
          _VANILLA_ATTENTION_NAME: {
              "k":
                  common_attention.split_heads(
                      tf.zeros(
                          [batch_size, attention_init_length, key_channels]),
                      hparams.num_heads),
              "v":
                  common_attention.split_heads(
                      tf.zeros(
                          [batch_size, attention_init_length, value_channels]),
                      hparams.num_heads),
          }
      } for layer in range(num_layers)
  })

  # Add branched layers. Pad with additional zeros for causal convolution.
  for layer in range(num_layers):
    cache["layer_%d" % layer][_CONV_BRANCHES_FIRST_LAYER_NAME] = tf.zeros([
        batch_size, attention_init_length + _DECODER_LEFT_CONV_PADDING,
        hparams.hidden_size
    ])
    cache["layer_%d" % layer][_CONV_BRANCHES_SECOND_LAYER_NAME] = tf.zeros([
        batch_size, attention_init_length + _DECODER_FINAL_CONV_PADDING,
        hparams.hidden_size * 2
    ])

  # Add encoder embedding attentions.
  if encoder_output is not None:
    cache = _add_attend_to_encoder_cache(
        cache=cache,
        attention_name=_FIRST_ATTEND_TO_ENCODER_NAME,
        hparams=hparams,
        num_layers=num_layers,
        key_channels=key_channels,
        value_channels=value_channels,
        vars_3d_num_heads=vars_3d_num_heads,
        scope_prefix=scope_prefix,
        encoder_output=encoder_output)
    cache = _add_attend_to_encoder_cache(
        cache=cache,
        attention_name=_SECOND_ATTEND_TO_ENCODER_NAME,
        hparams=hparams,
        num_layers=num_layers,
        key_channels=key_channels,
        value_channels=value_channels,
        vars_3d_num_heads=vars_3d_num_heads,
        scope_prefix=scope_prefix,
        encoder_output=encoder_output)

    cache["encoder_output"] = encoder_output
    cache["encoder_decoder_attention_bias"] = encoder_decoder_attention_bias

  return cache
403544
def attention_lm_moe_base_ae():
  """"""
  hparams = attention_lm_moe_base_long_seq()
  hparams.attention_type = AttentionType.LOCAL_EXPERTS

  hparams.learning_rate = 0.05
  hparams.learning_rate_warmup_steps = 10000
  # According to noam, ("n", "da") seems better for harder-to-learn models
  # hparams.layer_preprocess_sequence = "n"
  # hparams.layer_postprocess_sequence = "da"
  return hparams
232303
def build_graph(lstm_depth=3, batch_size=32, num_embedding=32, num_hidden=128,
                truncation_length=64, sample_length=1000, max_grad_norm=5,
                initial_learning_rate=0.1, reduce_learning_rate_multiplier=0.1,
                optimizer_epsilon=0.01):
  """"""

  # Get datasets.
  dataset_train = dataset_shakespeare.TinyShakespeareDataset(
      num_steps=truncation_length,
      batch_size=batch_size,
      subset="train",
      random=True,
      name="shake_train")

  dataset_valid = dataset_shakespeare.TinyShakespeareDataset(
      num_steps=truncation_length,
      batch_size=batch_size,
      subset="valid",
      random=False,
      name="shake_valid")

  dataset_test = dataset_shakespeare.TinyShakespeareDataset(
      num_steps=truncation_length,
      batch_size=batch_size,
      subset="test",
      random=False,
      name="shake_test")

  # Define model.
  model = TextModel(
      num_embedding=num_embedding,
      num_hidden=num_hidden,
      lstm_depth=lstm_depth,
      output_size=dataset_valid.vocab_size,
      use_dynamic_rnn=True,
      use_skip_connections=True)

  # Get the training loss.
  train_input_sequence, train_target_sequence = dataset_train()
  train_output_sequence_logits, train_final_state = model(train_input_sequence)  # pylint: disable=not-callable
  train_loss = dataset_train.cost(train_output_sequence_logits,
                                  train_target_sequence)

  # Get the validation loss.
  valid_input_sequence, valid_target_sequence = dataset_valid()
  valid_output_sequence_logits, _ = model(valid_input_sequence)  # pylint: disable=not-callable
  valid_loss = dataset_valid.cost(valid_output_sequence_logits,
                                  valid_target_sequence)

  # Get the test loss.
  test_input_sequence, test_target_sequence = dataset_test()
  test_output_sequence_logits, _ = model(test_input_sequence)  # pylint: disable=not-callable
  test_loss = dataset_test.cost(test_output_sequence_logits,
                                test_target_sequence)

  # Build graph to sample some strings during training.
  initial_logits = train_output_sequence_logits[truncation_length - 1]
  train_generated_string = model.generate_string(
      initial_logits=initial_logits,
      initial_state=train_final_state,
      sequence_length=sample_length)

  # Set up global norm clipping of gradients.
  trainable_variables = tf.trainable_variables()
  grads, _ = tf.clip_by_global_norm(
      tf.gradients(train_loss, trainable_variables), max_grad_norm)

  # Get learning rate and define annealing.
  learning_rate = tf.get_variable(
      "learning_rate",
      shape=[],
      dtype=tf.float32,
      initializer=tf.constant_initializer(initial_learning_rate),
      trainable=False)
  reduce_learning_rate = learning_rate.assign(
      learning_rate * reduce_learning_rate_multiplier)

  # Get training step counter.
  global_step = tf.get_variable(
      name="global_step",
      shape=[],
      dtype=tf.int64,
      initializer=tf.zeros_initializer(),
      trainable=False,
      collections=[tf.GraphKeys.GLOBAL_VARIABLES, tf.GraphKeys.GLOBAL_STEP])

  # Define optimizer and training step.
  optimizer = tf.train.AdamOptimizer(
      learning_rate, epsilon=optimizer_epsilon)
  train_step = optimizer.apply_gradients(
      zip(grads, trainable_variables),
      global_step=global_step)

  graph_tensors = {
      "train_loss": train_loss,
      "valid_loss": valid_loss,
      "test_loss": test_loss,
      "train_generated_string": train_generated_string,
      "reduce_learning_rate": reduce_learning_rate,
      "global_step": global_step,
      "train_step": train_step
  }

  # Return dataset_train for translation to human readable text.
  return graph_tensors, dataset_train
258909
def convert_simple_rnn(builder, layer, input_names, output_names, keras_layer):
    
    # Get input and output names
    hidden_size = keras_layer.units
    input_size = keras_layer.input_shape[-1]

    output_all = keras_layer.return_sequences
    reverse_input = keras_layer.go_backwards

    W_h = _np.zeros((hidden_size, hidden_size))
    W_x = _np.zeros((hidden_size, input_size))
    b = None

    implementation = keras_layer.implementation if hasattr(keras_layer,
            'implementation') else 0
    if implementation == 0:
        W_h = keras_layer.get_weights()[1].T
        W_x = keras_layer.get_weights()[0].T
        if keras_layer.use_bias:
            b = keras_layer.get_weights()[2]

    # Set actication type
    activation_str = _get_recurrent_activation_name_from_keras(keras_layer.activation)

    # Add to the network
    builder.add_simple_rnn(
        name = layer,
        W_h = W_h, W_x = W_x, b = b,
        hidden_size = hidden_size,
        input_size = input_size,
        activation = activation_str,
        input_names = input_names,
        output_names = output_names,
        output_all=output_all,
        reverse_input=reverse_input)
948691
def attention_lm_translation():
  """"""
  hparams = attention_lm_base()
  hparams.layer_preprocess_sequence = "n"
  hparams.layer_postprocess_sequence = "da"
  hparams.learning_rate = 0.4
  hparams.prepend_mode = "prepend_inputs_masked_attention"
  hparams.max_length = 512
  hparams.label_smoothing = 0.1
  hparams.shared_embedding_and_softmax_weights = True
  return hparams
462340
Word2VecModel train(Log log, TrainingProgressListener listener, Iterable<List<String>> sentences) throws InterruptedException {
		try (ProfilingTimer timer = ProfilingTimer.createLoggingSubtasks(log, "Training word2vec")) {
			final Multiset<String> counts;
			
			try (AC ac = timer.start("Acquiring word frequencies")) {
				listener.update(Stage.ACQUIRE_VOCAB, 0.0);
				counts = (vocab.isPresent())
							? vocab.get()
							: count(Iterables.concat(sentences));
			}
			
			final ImmutableMultiset<String> vocab;
			try (AC ac = timer.start("Filtering and sorting vocabulary")) {
				listener.update(Stage.FILTER_SORT_VOCAB, 0.0);
				vocab = filterAndSort(counts);
			}
			
			final Map<String, HuffmanNode> huffmanNodes;
			try (AC task = timer.start("Create Huffman encoding")) {
				huffmanNodes = new HuffmanCoding(vocab, listener).encode();
			}
			
			final NeuralNetworkModel model;
			try (AC task = timer.start("Training model %s", neuralNetworkConfig)) {
				model = neuralNetworkConfig.createTrainer(vocab, huffmanNodes, listener).train(sentences);
			}
			
			return new Word2VecModel(vocab.elementSet(), model.layerSize(), Doubles.concat(model.vectors()));
		}
	}
129904
def attention_lm_base():
  """"""
  hparams = common_hparams.basic_params1()
  hparams.hidden_size = 1024
  hparams.batch_size = 8192
  hparams.max_length = 256
  hparams.dropout = 0.0
  hparams.clip_grad_norm = 0.  # i.e. no gradient clipping
  hparams.optimizer_adam_epsilon = 1e-9
  hparams.learning_rate_decay_scheme = "noam"
  hparams.learning_rate = 0.1
  hparams.learning_rate_warmup_steps = 2000
  hparams.initializer_gain = 1.0
  hparams.num_hidden_layers = 6
  hparams.initializer = "uniform_unit_scaling"
  hparams.weight_decay = 0.0
  hparams.optimizer_adam_beta1 = 0.9
  hparams.optimizer_adam_beta2 = 0.98
  hparams.label_smoothing = 0.0
  hparams.shared_embedding_and_softmax_weights = False

  hparams.add_hparam("filter_size", 4096)  # Add new ones like this.
  # attention-related flags
  hparams.add_hparam("num_heads", 8)
  hparams.add_hparam("attention_key_channels", 0)
  hparams.add_hparam("attention_value_channels", 0)
  # All hyperparameters ending in "dropout" are automatically set to 0.0
  # when not in training mode.
  hparams.add_hparam("attention_dropout", 0.0)
  hparams.add_hparam("relu_dropout", 0.0)
  hparams.add_hparam("pos", "timing")  # timing, none
  hparams.add_hparam("encoder_full_attention", False)
  return hparams
1091467
def convert_lstm(builder, layer, input_names, output_names, keras_layer):
    

    hidden_size = keras_layer.output_dim
    input_size = keras_layer.input_shape[-1]
    if keras_layer.consume_less not in ['cpu', 'gpu']:
        raise ValueError('Cannot convert Keras layer with consume_less = %s' % keras_layer.consume_less)

    output_all = keras_layer.return_sequences
    reverse_input = keras_layer.go_backwards

    # Keras: I C F O; W_x, W_h, b
    # CoreML: I F O G; W_h and W_x are separated
    W_h, W_x, b = ([], [], [])
    if keras_layer.consume_less == 'cpu':
        W_h.append(keras_layer.get_weights()[1].T)
        W_h.append(keras_layer.get_weights()[7].T)
        W_h.append(keras_layer.get_weights()[10].T)
        W_h.append(keras_layer.get_weights()[4].T)

        W_x.append(keras_layer.get_weights()[0].T)
        W_x.append(keras_layer.get_weights()[6].T)
        W_x.append(keras_layer.get_weights()[9].T)
        W_x.append(keras_layer.get_weights()[3].T)

        b.append(keras_layer.get_weights()[2])
        b.append(keras_layer.get_weights()[8])
        b.append(keras_layer.get_weights()[11])
        b.append(keras_layer.get_weights()[5])
    else:
        keras_W_h = keras_layer.get_weights()[1].T
        W_h.append(keras_W_h[0 * hidden_size:][:hidden_size])
        W_h.append(keras_W_h[1 * hidden_size:][:hidden_size])
        W_h.append(keras_W_h[3 * hidden_size:][:hidden_size])
        W_h.append(keras_W_h[2 * hidden_size:][:hidden_size])

        keras_W_x = keras_layer.get_weights()[0].T
        W_x.append(keras_W_x[0 * hidden_size:][:hidden_size])
        W_x.append(keras_W_x[1 * hidden_size:][:hidden_size])
        W_x.append(keras_W_x[3 * hidden_size:][:hidden_size])
        W_x.append(keras_W_x[2 * hidden_size:][:hidden_size])

        keras_b = keras_layer.get_weights()[2]
        b.append(keras_b[0 * hidden_size:][:hidden_size])
        b.append(keras_b[1 * hidden_size:][:hidden_size])
        b.append(keras_b[3 * hidden_size:][:hidden_size])
        b.append(keras_b[2 * hidden_size:][:hidden_size])

    # Set activation type
    inner_activation_str = _get_recurrent_activation_name_from_keras(keras_layer.inner_activation)
    activation_str = _get_recurrent_activation_name_from_keras(keras_layer.activation)

    # Add to the network
    builder.add_unilstm(
        name = layer,
        W_h = W_h, W_x = W_x, b = b,
        hidden_size = hidden_size,
        input_size = input_size,
        input_names = input_names,
        output_names = output_names,
        inner_activation = inner_activation_str,
        cell_state_update_activation = activation_str,
        output_activation = activation_str,
        output_all = output_all,
        reverse_input = reverse_input)
234964
def convert_lstm(builder, layer, input_names, output_names, keras_layer):
    

    hidden_size = keras_layer.units
    input_size = keras_layer.input_shape[-1]
    output_all = keras_layer.return_sequences
    reverse_input = keras_layer.go_backwards

    # Keras: [W_x, W_h, b] each in I F C O
    # CoreML: I F O G; W_h and W_x are separated
    W_h, W_x, b = ([], [], [])
    keras_W_h = keras_layer.get_weights()[1].T
    W_h.append(keras_W_h[0 * hidden_size:][:hidden_size])
    W_h.append(keras_W_h[1 * hidden_size:][:hidden_size])
    W_h.append(keras_W_h[3 * hidden_size:][:hidden_size])
    W_h.append(keras_W_h[2 * hidden_size:][:hidden_size])

    keras_W_x = keras_layer.get_weights()[0].T
    W_x.append(keras_W_x[0 * hidden_size:][:hidden_size])
    W_x.append(keras_W_x[1 * hidden_size:][:hidden_size])
    W_x.append(keras_W_x[3 * hidden_size:][:hidden_size])
    W_x.append(keras_W_x[2 * hidden_size:][:hidden_size])
    if keras_layer.use_bias:
        keras_b = keras_layer.get_weights()[2]
        b.append(keras_b[0 * hidden_size:][:hidden_size])
        b.append(keras_b[1 * hidden_size:][:hidden_size])
        b.append(keras_b[3 * hidden_size:][:hidden_size])
        b.append(keras_b[2 * hidden_size:][:hidden_size])
    if len(b) == 0:
        b = None

    # Set activation type
    inner_activation_str = _get_recurrent_activation_name_from_keras(keras_layer.recurrent_activation)
    activation_str = _get_recurrent_activation_name_from_keras(keras_layer.activation)

    # Add to the network
    builder.add_unilstm(
        name = layer,
        W_h = W_h, W_x = W_x, b = b,
        hidden_size = hidden_size,
        input_size = input_size,
        input_names = input_names,
        output_names = output_names,
        inner_activation = inner_activation_str,
        cell_state_update_activation = activation_str,
        output_activation = activation_str,
        output_all = output_all,
        forget_bias = keras_layer.unit_forget_bias,
        reverse_input = reverse_input)
82677
def predict(self, x):
        
        recurrent_unit = self.config['recurrent-unit-type']
        
        # encode the input seq into a context vector
        if recurrent_unit == 'lstm':
            context_state = self.encoder.predict(np.array(x))
        elif recurrent_unit == 'gru':
            hidden_state = self.encoder.predict(np.array(x))
            context_state = [hidden_state]
        else:
            raise Exception('Invalid recurrent unit type: {}'.format(recurrent_unit))
        
        # create an empty target sequence, seeded with the start character
        y = self.vectorizer.vectorize_utterance([self.config['start']])
        response = []
        
        # i = 0
        while True:
            
            # decode the current sequence + current context into a
            # conditional distribution over next token:
            output_token_probs = None
            if recurrent_unit == 'lstm':
                output_token_probs, h, c = self.decoder.predict([y] + context_state)
                context_state = [h, c]
            elif recurrent_unit == 'gru':
                output_token_probs, hidden_state = self.decoder.predict([y] + context_state)
                context_state = [hidden_state]
            else:
                raise Exception('Invalid recurrent unit type: {}'.format(recurrent_unit))
            
            # sample a token from the output distribution (currently using maximum-likelihoo -- i.e., argmax)
            sampled_token = np.argmax(output_token_probs[0, -1, :])
            
            # add the sampled token to our output string
            response += [sampled_token]
            
            # exit condition: either we've
            # - hit the max length (self.data.output_max_len), or
            # - decoded a stop token ('\n')
            if (sampled_token == self.vectorizer.ie.transform([self.config['stop']]) or 
                len(response) >= self.config['max-utterance-length']):
                break
                
            # update the np array (target seq)
            y = np.array([sampled_token]) # np.concatenate((y, [sampled_token]))
            
        return response
103123
def learn(self, spiro, iterations=1):
        '''
        '''
        for i in range(iterations):
            self.stmem.train_cycle(spiro.obj.flatten())
131174
def next_frame_basic_recurrent():
  """"""
  hparams = basic_stochastic.next_frame_basic_stochastic_discrete()
  hparams.filter_double_steps = 2
  hparams.hidden_size = 64
  hparams.video_num_input_frames = 4
  hparams.video_num_target_frames = 4
  hparams.concat_internal_states = False
  hparams.add_hparam("num_lstm_layers", 2)
  hparams.add_hparam("num_lstm_filters", 256)
  return hparams
562500
def awd_lstm_lm_split(model:nn.Module) -> List[nn.Module]:
    ""
    groups = [[rnn, dp] for rnn, dp in zip(model[0].rnns, model[0].hidden_dps)]
    return groups + [[model[0].encoder, model[0].encoder_dp, model[1]]]