slice fused rnn weights
975495
def _slice_weights(self, arr, li, lh):
        """"""
        args = {}
        gate_names = self._gate_names
        directions = self._directions

        b = len(directions)
        p = 0
        for layer in range(self._num_layers):
            for direction in directions:
                for gate in gate_names:
                    name = '%s%s%d_i2h%s_weight'%(self._prefix, direction, layer, gate)
                    if layer > 0:
                        size = b*lh*lh
                        args[name] = arr[p:p+size].reshape((lh, b*lh))
                    else:
                        size = li*lh
                        args[name] = arr[p:p+size].reshape((lh, li))
                    p += size
                for gate in gate_names:
                    name = '%s%s%d_h2h%s_weight'%(self._prefix, direction, layer, gate)
                    size = lh**2
                    args[name] = arr[p:p+size].reshape((lh, lh))
                    p += size

        for layer in range(self._num_layers):
            for direction in directions:
                for gate in gate_names:
                    name = '%s%s%d_i2h%s_bias'%(self._prefix, direction, layer, gate)
                    args[name] = arr[p:p+lh]
                    p += lh
                for gate in gate_names:
                    name = '%s%s%d_h2h%s_bias'%(self._prefix, direction, layer, gate)
                    args[name] = arr[p:p+lh]
                    p += lh

        assert p == arr.size, "Invalid parameters size for FusedRNNCell"
        return args
508353
def wider_bn(layer, start_dim, total_dim, n_add, weighted=True):
    '''
    '''
    n_dim = get_n_dim(layer)
    if not weighted:
        return get_batch_norm_class(n_dim)(layer.num_features + n_add)

    weights = layer.get_weights()

    new_weights = [
        add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])),
        add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])),
        add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])),
        add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])),
    ]

    student_w = tuple()
    for weight, new_weight in zip(weights, new_weights):
        temp_w = weight.copy()
        temp_w = np.concatenate(
            (temp_w[:start_dim], new_weight, temp_w[start_dim:total_dim])
        )
        student_w += (temp_w,)
    new_layer = get_batch_norm_class(n_dim)(layer.num_features + n_add)
    new_layer.set_weights(student_w)
    return new_layer
740258
def _create_rnn_cells(self):
        """"""
        lstm_cell = tf.nn.rnn_cell.LSTMCell(
            self.num_hidden, forget_bias=0.0)
        lstm_cell = tf.nn.rnn_cell.DropoutWrapper(
            lstm_cell, output_keep_prob=self.dropout)
        self.cell = tf.nn.rnn_cell.MultiRNNCell(
            [lstm_cell] * self.num_layers)
1184594
def lstm(num_hidden, indata, prev_state, param, seqidx, layeridx, dropout=0., num_hidden_proj=0, is_batchnorm=False,
         gamma=None, beta=None, name=None):
    """"""
    # dropout input
    if dropout > 0.:
        indata = mx.sym.Dropout(data=indata, p=dropout)

    i2h = mx.sym.FullyConnected(data=indata,
                                weight=param.i2h_weight,
                                bias=param.i2h_bias,
                                num_hidden=num_hidden * 4,
                                name="t%d_l%d_i2h" % (seqidx, layeridx))
    if is_batchnorm:
        if name is not None:
            i2h = batchnorm(net=i2h, gamma=gamma, beta=beta, name="%s_batchnorm" % name)
        else:
            i2h = batchnorm(net=i2h, gamma=gamma, beta=beta)

    h2h = mx.sym.FullyConnected(data=prev_state.h,
                                weight=param.h2h_weight,
                                # bias=param.h2h_bias,
                                no_bias=True,
                                num_hidden=num_hidden * 4,
                                name="t%d_l%d_h2h" % (seqidx, layeridx))

    gates = i2h + h2h
    slice_gates = mx.sym.SliceChannel(gates, num_outputs=4,
                                      name="t%d_l%d_slice" % (seqidx, layeridx))

    Wcidc = mx.sym.broadcast_mul(param.c2i_bias, prev_state.c) + slice_gates[0]
    in_gate = mx.sym.Activation(Wcidc, act_type="sigmoid")

    in_transform = mx.sym.Activation(slice_gates[1], act_type="tanh")

    Wcfdc = mx.sym.broadcast_mul(param.c2f_bias, prev_state.c) + slice_gates[2]
    forget_gate = mx.sym.Activation(Wcfdc, act_type="sigmoid")

    next_c = (forget_gate * prev_state.c) + (in_gate * in_transform)

    Wcoct = mx.sym.broadcast_mul(param.c2o_bias, next_c) + slice_gates[3]
    out_gate = mx.sym.Activation(Wcoct, act_type="sigmoid")

    next_h = out_gate * mx.sym.Activation(next_c, act_type="tanh")

    if num_hidden_proj > 0:
        proj_next_h = mx.sym.FullyConnected(data=next_h,
                                            weight=param.ph2h_weight,
                                            no_bias=True,
                                            num_hidden=num_hidden_proj,
                                            name="t%d_l%d_ph2h" % (seqidx, layeridx))

        return LSTMState(c=next_c, h=proj_next_h)
    else:
        return LSTMState(c=next_c, h=next_h)
1337142
def iter_slices(string, slice_length):
    """"""
    pos = 0
    if slice_length is None or slice_length <= 0:
        slice_length = len(string)
    while pos < len(string):
        yield string[pos:pos + slice_length]
        pos += slice_length
467802
def convert_gru(builder, layer, input_names, output_names, keras_layer):
    

    hidden_size = keras_layer.units
    input_size = keras_layer.input_shape[-1]

    output_all = keras_layer.return_sequences
    reverse_input = keras_layer.go_backwards

    # Keras: Z R O
    # CoreML: Z R O
    W_h, W_x, b = ([], [], [])
    keras_W_h = keras_layer.get_weights()[1].T
    W_h.append(keras_W_h[0 * hidden_size:][:hidden_size])
    W_h.append(keras_W_h[1 * hidden_size:][:hidden_size])
    W_h.append(keras_W_h[2 * hidden_size:][:hidden_size])

    keras_W_x = keras_layer.get_weights()[0].T
    W_x.append(keras_W_x[0 * hidden_size:][:hidden_size])
    W_x.append(keras_W_x[1 * hidden_size:][:hidden_size])
    W_x.append(keras_W_x[2 * hidden_size:][:hidden_size])

    if keras_layer.use_bias:
        keras_b = keras_layer.get_weights()[2]
        b.append(keras_b[0 * hidden_size:][:hidden_size])
        b.append(keras_b[1 * hidden_size:][:hidden_size])
        b.append(keras_b[2 * hidden_size:][:hidden_size])
    if len(b) == 0:
        b = None

    # Set actication type
    inner_activation_str = _get_recurrent_activation_name_from_keras(keras_layer.recurrent_activation)
    activation_str = _get_recurrent_activation_name_from_keras(keras_layer.activation)

    # Add to the network
    builder.add_gru(
       name = layer,
       W_h = W_h, W_x = W_x, b = b,
       input_size = input_size,
       hidden_size = hidden_size,
       input_names = input_names,
       output_names = output_names,
       activation = activation_str,
       inner_activation = inner_activation_str,
       output_all = output_all,
       reverse_input = reverse_input)
258909
def convert_simple_rnn(builder, layer, input_names, output_names, keras_layer):
    
    # Get input and output names
    hidden_size = keras_layer.units
    input_size = keras_layer.input_shape[-1]

    output_all = keras_layer.return_sequences
    reverse_input = keras_layer.go_backwards

    W_h = _np.zeros((hidden_size, hidden_size))
    W_x = _np.zeros((hidden_size, input_size))
    b = None

    implementation = keras_layer.implementation if hasattr(keras_layer,
            'implementation') else 0
    if implementation == 0:
        W_h = keras_layer.get_weights()[1].T
        W_x = keras_layer.get_weights()[0].T
        if keras_layer.use_bias:
            b = keras_layer.get_weights()[2]

    # Set actication type
    activation_str = _get_recurrent_activation_name_from_keras(keras_layer.activation)

    # Add to the network
    builder.add_simple_rnn(
        name = layer,
        W_h = W_h, W_x = W_x, b = b,
        hidden_size = hidden_size,
        input_size = input_size,
        activation = activation_str,
        input_names = input_names,
        output_names = output_names,
        output_all=output_all,
        reverse_input=reverse_input)
403936
def sym_gen(seq_len):
    
    sentence_shape = train_iter.provide_data[0][1]
    char_sentence_shape = train_iter.provide_data[1][1]
    entities_shape = train_iter.provide_label[0][1]

    X_sent = mx.symbol.Variable(train_iter.provide_data[0].name)
    X_char_sent = mx.symbol.Variable(train_iter.provide_data[1].name)
    Y = mx.sym.Variable(train_iter.provide_label[0].name)

    ###############################
    # Character embedding component
    ###############################
    char_embeddings = mx.sym.Embedding(data=X_char_sent, input_dim=len(char_to_index), output_dim=args.char_embed, name='char_embed')
    char_embeddings = mx.sym.reshape(data=char_embeddings, shape=(0,1,seq_len,-1,args.char_embed), name='char_embed2')

    char_cnn_outputs = []
    for i, filter_size in enumerate(args.char_filter_list):
        # Kernel that slides over entire words resulting in a 1d output
        convi = mx.sym.Convolution(data=char_embeddings, kernel=(1, filter_size, args.char_embed), stride=(1, 1, 1),
                                   num_filter=args.char_filters, name="char_conv_layer_" + str(i))
        acti = mx.sym.Activation(data=convi, act_type='tanh')
        pooli = mx.sym.Pooling(data=acti, pool_type='max', kernel=(1, char_sentence_shape[2] - filter_size + 1, 1),
                               stride=(1, 1, 1), name="char_pool_layer_" + str(i))
        pooli = mx.sym.transpose(mx.sym.Reshape(pooli, shape=(0, 0, 0)), axes=(0, 2, 1), name="cchar_conv_layer_" + str(i))
        char_cnn_outputs.append(pooli)

    # combine features from all filters & apply dropout
    cnn_char_features = mx.sym.Concat(*char_cnn_outputs, dim=2, name="cnn_char_features")
    regularized_cnn_char_features = mx.sym.Dropout(data=cnn_char_features, p=args.dropout, mode='training',
                                                   name='regularized charCnn features')

    ##################################
    # Combine char and word embeddings
    ##################################
    word_embeddings = mx.sym.Embedding(data=X_sent, input_dim=len(word_to_index), output_dim=args.word_embed, name='word_embed')
    rnn_features = mx.sym.Concat(*[word_embeddings, regularized_cnn_char_features], dim=2, name='rnn input')

    ##############################
    # Bidirectional LSTM component
    ##############################

    # unroll the lstm cell in time, merging outputs
    bi_cell.reset()
    output, states = bi_cell.unroll(length=seq_len, inputs=rnn_features, merge_outputs=True)

    # Map to num entity classes
    rnn_output = mx.sym.Reshape(output, shape=(-1, args.lstm_state_size * 2), name='r_output')
    fc = mx.sym.FullyConnected(data=rnn_output, num_hidden=len(entity_to_index), name='fc_layer')

    # reshape back to same shape as loss will be
    reshaped_fc = mx.sym.transpose(mx.sym.reshape(fc, shape=(-1, seq_len, len(entity_to_index))), axes=(0, 2, 1))
    sm = mx.sym.SoftmaxOutput(data=reshaped_fc, label=Y, ignore_label=-1, use_ignore=True, multi_output=True, name='softmax')
    return sm, [v.name for v in train_iter.provide_data], [v.name for v in train_iter.provide_label]
1332651
def value_net(rng_key,
              batch_observations_shape,
              num_actions,
              bottom_layers=None):
  """"""
  del num_actions

  if bottom_layers is None:
    bottom_layers = []
  bottom_layers.extend([
      layers.Dense(1),
  ])
  net = layers.Serial(*bottom_layers)
  return net.initialize(batch_observations_shape, rng_key), net
494489
def generator(ngf, nc, no_bias=True, fix_gamma=True, eps=1e-5 + 1e-12, z_dim=100, activation='sigmoid'):
    '''
    '''
    BatchNorm = mx.sym.BatchNorm
    rand = mx.sym.Variable('rand')

    rand = mx.sym.Reshape(rand, shape=(-1, z_dim, 1, 1))

    g1 = mx.sym.Deconvolution(rand, name='gen1', kernel=(5,5), stride=(2,2),target_shape=(2,2), num_filter=ngf*8, no_bias=no_bias)
    gbn1 = BatchNorm(g1, name='genbn1', fix_gamma=fix_gamma, eps=eps)
    gact1 = mx.sym.Activation(gbn1, name="genact1", act_type="relu")

    g2 = mx.sym.Deconvolution(gact1, name='gen2', kernel=(5,5), stride=(2,2),target_shape=(4,4), num_filter=ngf*4, no_bias=no_bias)
    gbn2 = BatchNorm(g2, name='genbn2', fix_gamma=fix_gamma, eps=eps)
    gact2 = mx.sym.Activation(gbn2, name='genact2', act_type='relu')

    g3 = mx.sym.Deconvolution(gact2, name='gen3', kernel=(5,5), stride=(2,2), target_shape=(8,8), num_filter=ngf*2, no_bias=no_bias)
    gbn3 = BatchNorm(g3, name='genbn3', fix_gamma=fix_gamma, eps=eps)
    gact3 = mx.sym.Activation(gbn3, name='genact3', act_type='relu')

    g4 = mx.sym.Deconvolution(gact3, name='gen4', kernel=(5,5), stride=(2,2), target_shape=(16,16), num_filter=ngf, no_bias=no_bias)
    gbn4 = BatchNorm(g4, name='genbn4', fix_gamma=fix_gamma, eps=eps)
    gact4 = mx.sym.Activation(gbn4, name='genact4', act_type='relu')

    g5 = mx.sym.Deconvolution(gact4, name='gen5', kernel=(5,5), stride=(2,2), target_shape=(32,32), num_filter=nc, no_bias=no_bias)
    gout = mx.sym.Activation(g5, name='genact5', act_type=activation)

    return gout
701201
function slice(v, start, end, step) {
  var r = [];
  var i = void 0;
  if (step > 0) {
    for (i = start; i < end; i += step) {
      r.push(v[i]);
    }
  } else {
    for (i = start; i > end; i += step) {
      r.push(v[i]);
    }
  }
  return r;
}
480775
def vdcnn(embedding_size, maxlen, max_features,
          filter_nr, kernel_size, repeat_block, dense_size, repeat_dense, output_size, output_activation,
          max_pooling, mean_pooling, weighted_average_attention, concat_mode,
          dropout_embedding, conv_dropout, dense_dropout, dropout_mode,
          conv_kernel_reg_l2, conv_bias_reg_l2,
          dense_kernel_reg_l2, dense_bias_reg_l2,
          use_prelu, use_batch_norm, batch_norm_first):
    

    input_text = Input(shape=(maxlen,))
    x = Embedding(input_dim=max_features, output_dim=embedding_size)(input_text)

    x = dropout_block(dropout_embedding, dropout_mode)(x)

    x = convolutional_block(filter_nr, kernel_size, use_batch_norm, use_prelu, conv_dropout, dropout_mode,
                            conv_kernel_reg_l2, conv_bias_reg_l2, batch_norm_first)(x)

    for i in range(repeat_block):
        if i + 1 != repeat_block:
            x = vdcnn_block(filter_nr, kernel_size, use_batch_norm, use_prelu, conv_dropout, dropout_mode,
                            conv_kernel_reg_l2, conv_bias_reg_l2, batch_norm_first, last_block=False)(x)
        else:
            x = vdcnn_block(filter_nr, kernel_size, use_batch_norm, use_prelu, conv_dropout, dropout_mode,
                            conv_kernel_reg_l2, conv_bias_reg_l2, batch_norm_first, last_block=True)(x)

    predictions = classification_block(dense_size=dense_size, repeat_dense=repeat_dense,
                                       output_size=output_size, output_activation=output_activation,
                                       max_pooling=max_pooling,
                                       mean_pooling=mean_pooling,
                                       weighted_average_attention=weighted_average_attention,
                                       concat_mode=concat_mode,
                                       dropout=dense_dropout,
                                       kernel_reg_l2=dense_kernel_reg_l2, bias_reg_l2=dense_bias_reg_l2,
                                       use_prelu=use_prelu, use_batch_norm=use_batch_norm,
                                       batch_norm_first=batch_norm_first)(x)
    model = Model(inputs=input_text, outputs=predictions)
    return model
1367062
def sg_rnn_layer_func(func):
    
    """
    @wraps(func)
    def wrapper(tensor, **kwargs):
        r"""Manages arguments of `tf.sg_opt`.

        Args:
          tensor: automatically passed by decorator
          kwargs:
              in_dim: An integer. The size of input dimension, which is set to the last one by default.
              dim: An integer. The size of output dimension. Has the same value as in_dim by default.
              ln: Boolean. If True, layer normalization is applied.
              bias: Boolean. If True, biases are added. As a default, it is set to True
              name: A name for the layer. As a default, the function name is assigned.
              reuse: `True` or `None`; if `True`, we go into reuse mode for this `layer` scope
                as well as all sub-scopes; if `None`, we just inherit the parent scope reuse.
              summary: If True, summaries are added. The default is True.
        """

        # kwargs parsing
        opt = tf.sg_opt(kwargs) + sg_get_context()

        # set default argument
        try:
            shape = tensor.get_shape().as_list()
            # dropout off
            opt += tf.sg_opt(shape=shape, in_dim=shape[-1], dim=shape[-1], dout=0, summary=True)
            # disable bias when normalization on
            opt += tf.sg_opt(bias=not opt.ln)
        finally:
            pass

        # automatic layer naming
        if opt.name is None:

            # layer function name will be used as layer name
            opt.name = func.__name__.replace('sg_', '')

            # find existing layer names
            exist_layers = []
            for t in tf.global_variables():
                scope_name = tf.get_variable_scope().name
                prefix = scope_name + '/' if len(scope_name) > 0 else ''
                i = t.name.rfind(prefix + opt.name)
                if i >= 0:
                    exist_layers.append(t.name[i:].split('/')[-2])
            exist_layers = list(set(exist_layers))

            # layer name numbering
            if len(exist_layers) == 0:
                opt.name += '_1'
            else:
                opt.name += '_%d' % (max([int(n.split('_')[-1]) for n in exist_layers]) + 1)

        with tf.variable_scope(opt.name, reuse=opt.reuse) as scope:

            # call layer function
            out = func(tensor, opt)

            # apply dropout
            if opt.dout:
                out = tf.cond(_phase,
                              lambda: tf.nn.dropout(out, 1 - opt.dout),
                              lambda: out)

            # rename tensor
            out = tf.identity(out, 'out')

            # add final output summary
            if opt.summary:
                tf.sg_summary_activation(out)

            # save node info for reuse
            out._sugar = tf.sg_opt(func=func, arg=tf.sg_opt(kwargs) + sg_get_context(),
                                   prev=tensor, is_layer=True, name=opt.name)
            # inject reuse function
            out.sg_reuse = types.MethodType(sg_reuse, out)

        return out

    return wrapper
88438
def collate_fn(batch, train=True):
    
    premise_batch, _ = pad_batch([row['premise'] for row in batch])
    hypothesis_batch, _ = pad_batch([row['hypothesis'] for row in batch])
    label_batch = torch.stack([row['label'] for row in batch])

    # PyTorch RNN requires batches to be transposed for speed and integration with CUDA
    transpose = (lambda b: b.t_().squeeze(0).contiguous())

    return (transpose(premise_batch), transpose(hypothesis_batch), transpose(label_batch))
527066
def get_recurrent_encoder(config: RecurrentEncoderConfig, prefix: str) -> 'Encoder':
    
    # TODO give more control on encoder architecture
    encoder_seq = EncoderSequence([], config.dtype)

    if config.conv_config is not None:
        encoder_seq.append(ConvolutionalEmbeddingEncoder, config=config.conv_config,
                           prefix=prefix + C.CHAR_SEQ_ENCODER_PREFIX)
        if config.conv_config.add_positional_encoding:
            # If specified, add positional encodings to segment embeddings
            encoder_seq.append(AddSinCosPositionalEmbeddings,
                               num_embed=config.conv_config.num_embed,
                               scale_up_input=False,
                               scale_down_positions=False,
                               prefix="%s%sadd_positional_encodings" % (prefix, C.CHAR_SEQ_ENCODER_PREFIX))
        encoder_seq.append(ConvertLayout, infer_hidden=True, target_layout=C.TIME_MAJOR)
    else:
        encoder_seq.append(ConvertLayout, target_layout=C.TIME_MAJOR, num_hidden=0)

    if config.reverse_input:
        encoder_seq.append(ReverseSequence, infer_hidden=True)

    if config.rnn_config.residual:
        utils.check_condition(config.rnn_config.first_residual_layer >= 2,
                              "Residual connections on the first encoder layer are not supported")

    # One layer bi-directional RNN:
    encoder_seq.append(BiDirectionalRNNEncoder,
                       rnn_config=config.rnn_config.copy(num_layers=1),
                       prefix=prefix + C.BIDIRECTIONALRNN_PREFIX,
                       layout=C.TIME_MAJOR)

    if config.rnn_config.num_layers > 1:
        # Stacked uni-directional RNN:
        # Because we already have a one layer bi-rnn we reduce the num_layers as well as the first_residual_layer.
        remaining_rnn_config = config.rnn_config.copy(num_layers=config.rnn_config.num_layers - 1,
                                                      first_residual_layer=config.rnn_config.first_residual_layer - 1)
        encoder_seq.append(RecurrentEncoder,
                           rnn_config=remaining_rnn_config,
                           prefix=prefix + C.STACKEDRNN_PREFIX,
                           layout=C.TIME_MAJOR)

    encoder_seq.append(ConvertLayout, infer_hidden=True, target_layout=C.BATCH_MAJOR)

    return encoder_seq
751800
def get_lenet():
    
    data = mx.symbol.Variable('data')

    # first conv
    conv1 = mx.symbol.CaffeOp(data_0=data, num_weight=2,
                              prototxt="layer{type:\"Convolution\" "
                                       "convolution_param { num_output: 20 kernel_size: 5 stride: 1} }")
    act1 = mx.symbol.CaffeOp(data_0=conv1, prototxt="layer{type:\"TanH\"}")
    pool1 = mx.symbol.CaffeOp(data_0=act1,
                              prototxt="layer{type:\"Pooling\" pooling_param { pool: MAX kernel_size: 2 stride: 2}}")

    # second conv
    conv2 = mx.symbol.CaffeOp(data_0=pool1, num_weight=2,
                              prototxt="layer{type:\"Convolution\" "
                                       "convolution_param { num_output: 50 kernel_size: 5 stride: 1} }")
    act2 = mx.symbol.CaffeOp(data_0=conv2, prototxt="layer{type:\"TanH\"}")
    pool2 = mx.symbol.CaffeOp(data_0=act2,
                              prototxt="layer{type:\"Pooling\" pooling_param { pool: MAX kernel_size: 2 stride: 2}}")

    fc1 = mx.symbol.CaffeOp(data_0=pool2, num_weight=2,
                            prototxt="layer{type:\"InnerProduct\" inner_product_param{num_output: 500} }")
    act3 = mx.symbol.CaffeOp(data_0=fc1, prototxt="layer{type:\"TanH\"}")

    # second fullc
    fc2 = mx.symbol.CaffeOp(data_0=act3, num_weight=2,
                            prototxt="layer{type:\"InnerProduct\"inner_product_param{num_output: 10} }")
    if use_caffe_loss:
        label = mx.symbol.Variable('softmax_label')
        lenet = mx.symbol.CaffeLoss(data=fc2, label=label, grad_scale=1, name='softmax',
                                    prototxt="layer{type:\"SoftmaxWithLoss\"}")
    else:
        lenet = mx.symbol.SoftmaxOutput(data=fc2, name='softmax')
    return lenet
1307924
def batch_norm_relu(inputs, is_training, relu=True):
  """"""
  inputs = mtf.layers.batch_norm(
      inputs,
      is_training,
      BATCH_NORM_DECAY,
      epsilon=BATCH_NORM_EPSILON,
      init_zero=(not relu))
  if relu:
    inputs = mtf.relu(inputs)
  return inputs