keras dropout layer.
429072
def keras_dropout(layer, rate):
    '''
    '''

    from keras import layers

    input_dim = len(layer.input.shape)
    if input_dim == 2:
        return layers.SpatialDropout1D(rate)
    elif input_dim == 3:
        return layers.SpatialDropout2D(rate)
    elif input_dim == 4:
        return layers.SpatialDropout3D(rate)
    else:
        return layers.Dropout(rate)
1189232
def DiscriminatorLayerLoss():
    '''
    '''

    data = mx.sym.Variable('data')

    label = mx.sym.Variable('label')

    data = mx.sym.Flatten(data)
    label = mx.sym.Flatten(label)

    label = mx.sym.BlockGrad(label)

    zeros = mx.sym.zeros_like(data)

    output = -GaussianLogDensity(label, data, zeros)

    dloss = mx.symbol.MakeLoss(mx.symbol.mean(output),name='lloss')

    return dloss
531584
def defuse_activation(self):
        
        idx, nb_layers = 0, len(self.layer_list)
        while idx < nb_layers:
            layer = self.layer_list[idx]
            k_layer = self.keras_layer_map[layer]
            # unwrap time-distributed layers
            if (isinstance(k_layer, _keras.layers.TimeDistributed)):
                k_layer = k_layer.layer
            if (isinstance(k_layer, _keras.layers.convolutional.Convolution2D) or
                isinstance(k_layer, _keras.layers.convolutional.Convolution1D) or
                isinstance(k_layer, _keras.layers.core.Dense)):

                import six
                if six.PY2:
                    func_name = k_layer.activation.func_name
                else:
                    func_name = k_layer.activation.__name__

                if (func_name != 'linear'):
                    # Create new layer
                    new_layer = layer + '__activation__'
                    new_keras_layer = _keras.layers.core.Activation(func_name)
                    # insert new layer after it
                    self._insert_layer_after(idx, new_layer, new_keras_layer)
                    idx += 1
                    nb_layers += 1

            idx += 1
595933
def discharge(self):
        """"""
        Q = np.zeros(self.aq.naq)
        Q[self.layers] = self.parameters[:, 0]
        return Q
205712
def _setweights(self):
        ""
        for layer in self.layer_names:
            raw_w = getattr(self, f'{layer}_raw')
            self.module._parameters[layer] = F.dropout(raw_w, p=self.weight_p, training=self.training)
14925
def disable_dropout(self):
        
        for attr, val in self.__dict__.items():
            if isinstance(val, Config):
                val.disable_dropout()
            elif 'dropout' in attr and isinstance(val, float):
                logger.debug("Setting %s to 0.0", attr)
                setattr(self, attr, 0.0)
148248
def _get_rnn_layer(mode, num_layers, input_size, hidden_size, dropout, weight_dropout):
    """"""
    if mode == 'rnn_relu':
        rnn_block = functools.partial(rnn.RNN, activation='relu')
    elif mode == 'rnn_tanh':
        rnn_block = functools.partial(rnn.RNN, activation='tanh')
    elif mode == 'lstm':
        rnn_block = rnn.LSTM
    elif mode == 'gru':
        rnn_block = rnn.GRU

    block = rnn_block(hidden_size, num_layers, dropout=dropout,
                      input_size=input_size)

    if weight_dropout:
        apply_weight_drop(block, '.*h2h_weight', rate=weight_dropout)

    return block
613592
def bn_drop_lin(n_in:int, n_out:int, bn:bool=True, p:float=0., actn:Optional[nn.Module]=None):
    ""
    layers = [nn.BatchNorm1d(n_in)] if bn else []
    if p != 0: layers.append(nn.Dropout(p))
    layers.append(nn.Linear(n_in, n_out))
    if actn is not None: layers.append(actn)
    return layers
38055
def dropout_no_scaling(x, keep_prob):
  
  if keep_prob == 1.0:
    return x
  mask = tf.less(tf.random_uniform(tf.shape(x)), keep_prob)
  return x * cast_like(mask, x)
212493
def _remove_layer(self, layer):
        
        successors = self.get_successors(layer)
        predecessors = self.get_predecessors(layer)
        # remove all edges
        for succ in successors:
            self._remove_edge(layer, succ)
        for pred in predecessors:
            self._remove_edge(pred, layer)
        # remove layer in the data structures
        self.keras_layer_map.pop(layer)
        self.layer_list.remove(layer)
325696
def deeper_conv_block(conv_layer, kernel_size, weighted=True):
    '''
    '''
    n_dim = get_n_dim(conv_layer)
    filter_shape = (kernel_size,) * 2
    n_filters = conv_layer.filters
    weight = np.zeros((n_filters, n_filters) + filter_shape)
    center = tuple(map(lambda x: int((x - 1) / 2), filter_shape))
    for i in range(n_filters):
        filter_weight = np.zeros((n_filters,) + filter_shape)
        index = (i,) + center
        filter_weight[index] = 1
        weight[i, ...] = filter_weight
    bias = np.zeros(n_filters)
    new_conv_layer = get_conv_class(n_dim)(
        conv_layer.filters, n_filters, kernel_size=kernel_size
    )
    bn = get_batch_norm_class(n_dim)(n_filters)

    if weighted:
        new_conv_layer.set_weights(
            (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1])))
        )
        new_weights = [
            add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])),
            add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])),
            add_noise(np.zeros(n_filters, dtype=np.float32), np.array([0, 1])),
            add_noise(np.ones(n_filters, dtype=np.float32), np.array([0, 1])),
        ]
        bn.set_weights(new_weights)

    return [StubReLU(), new_conv_layer, bn]
490965
def defuse_activation(self):
        
        idx, nb_layers = 0, len(self.layer_list)
        while idx < nb_layers:
            layer = self.layer_list[idx]
            k_layer = self.keras_layer_map[layer]
            if (isinstance(k_layer, _keras.layers.TimeDistributed)):
                k_layer = k_layer.layer
            if (isinstance(k_layer, _keras.layers.Conv2D) or
                isinstance(k_layer, _keras.layers.Conv1D) or
                isinstance(k_layer, _keras.layers.SeparableConv2D) or
                isinstance(k_layer, _keras.layers.SeparableConv1D) or
                isinstance(k_layer, _keras.layers.Dense)):

                func_name = k_layer.activation.func_name if six.PY2 else \
                        k_layer.activation.__name__

                if func_name != 'linear':
                    # Create new layer
                    new_layer = layer + '__activation__'
                    new_keras_layer = _keras.layers.core.Activation(func_name)
                    # insert new layer after it
                    self._insert_layer_after(idx, new_layer, new_keras_layer)
                    idx += 1
                    nb_layers += 1
            idx += 1
921105
private void backpropagate(Layer upper, Layer lower) {
        for (int i = 0; i <= lower.units; i++) {
            double out = lower.output[i];
            double err = 0;
            for (int j = 0; j < upper.units; j++) {
                err += upper.weight[j][i] * upper.error[j];
            }
            if (activationFunction==ActivationFunction.LOGISTIC_SIGMOID) {
                lower.error[i] = out * (1.0 - out) * err;
            }
            else if (activationFunction==ActivationFunction.TANH){
                lower.error[i] = (1-(out*out))*err;
            }
        }
    }
280471
def shake_shake_layer(x, output_filters, num_blocks, stride, hparams):
  """"""
  for block_num in range(num_blocks):
    curr_stride = stride if (block_num == 0) else 1
    with tf.variable_scope("layer_{}".format(block_num)):
      x = shake_shake_block(x, output_filters, curr_stride, hparams)
  return x
508353
def wider_bn(layer, start_dim, total_dim, n_add, weighted=True):
    '''
    '''
    n_dim = get_n_dim(layer)
    if not weighted:
        return get_batch_norm_class(n_dim)(layer.num_features + n_add)

    weights = layer.get_weights()

    new_weights = [
        add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])),
        add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])),
        add_noise(np.zeros(n_add, dtype=np.float32), np.array([0, 1])),
        add_noise(np.ones(n_add, dtype=np.float32), np.array([0, 1])),
    ]

    student_w = tuple()
    for weight, new_weight in zip(weights, new_weights):
        temp_w = weight.copy()
        temp_w = np.concatenate(
            (temp_w[:start_dim], new_weight, temp_w[start_dim:total_dim])
        )
        student_w += (temp_w,)
    new_layer = get_batch_norm_class(n_dim)(layer.num_features + n_add)
    new_layer.set_weights(student_w)
    return new_layer
789114
def loss_batch(model:nn.Module, xb:Tensor, yb:Tensor, loss_func:OptLossFunc=None, opt:OptOptimizer=None,
               cb_handler:Optional[CallbackHandler]=None)->Tuple[Union[Tensor,int,float,str]]:
    ""
    cb_handler = ifnone(cb_handler, CallbackHandler())
    if not is_listy(xb): xb = [xb]
    if not is_listy(yb): yb = [yb]
    out = model(*xb)
    out = cb_handler.on_loss_begin(out)

    if not loss_func: return to_detach(out), yb[0].detach()
    loss = loss_func(out, *yb)

    if opt is not None:
        loss,skip_bwd = cb_handler.on_backward_begin(loss)
        if not skip_bwd:                     loss.backward()
        if not cb_handler.on_backward_end(): opt.step()
        if not cb_handler.on_step_end():     opt.zero_grad()

    return loss.detach().cpu()
1208089
def model(dropout, vocab, model_mode, output_size):
    """"""

    textCNN = SentimentNet(dropout=dropout, vocab_size=len(vocab), model_mode=model_mode,\
                       output_size=output_size)
    textCNN.hybridize()
    return textCNN