Map Keras to DL4J weight initialization functions.

@param kerasInit String containing Keras initialization function name
@return DL4J weight initialization enum
@see WeightInit
327506
public static Pair<WeightInit, Distribution> mapWeightInitialization(String kerasInit,
                                                                         KerasLayerConfiguration conf,
                                                                         Map<String, Object> initConfig,
                                                                         int kerasMajorVersion)
            throws UnsupportedKerasConfigurationException, InvalidKerasConfigurationException {


        // TODO: Identity and VarianceScaling need "scale" factor
        WeightInit init = null;
        Distribution dist = null;
        if (kerasInit != null) {
            if (kerasInit.equals(conf.getINIT_GLOROT_NORMAL()) ||
                    kerasInit.equals(conf.getINIT_GLOROT_NORMAL_ALIAS())) {
                init = WeightInit.XAVIER;
            } else if (kerasInit.equals(conf.getINIT_GLOROT_UNIFORM()) ||
                    kerasInit.equals(conf.getINIT_GLOROT_UNIFORM_ALIAS())) {
                init = WeightInit.XAVIER_UNIFORM;
            } else if (kerasInit.equals(conf.getINIT_LECUN_NORMAL()) ||
                    kerasInit.equals(conf.getINIT_LECUN_NORMAL_ALIAS())) {
                init = WeightInit.LECUN_NORMAL;
            } else if (kerasInit.equals(conf.getINIT_LECUN_UNIFORM()) ||
                    kerasInit.equals(conf.getINIT_LECUN_UNIFORM_ALIAS())) {
                init = WeightInit.LECUN_UNIFORM;
            } else if (kerasInit.equals(conf.getINIT_HE_NORMAL()) ||
                    kerasInit.equals(conf.getINIT_HE_NORMAL_ALIAS())) {
                init = WeightInit.RELU;
            } else if (kerasInit.equals(conf.getINIT_HE_UNIFORM()) ||
                    kerasInit.equals(conf.getINIT_HE_UNIFORM_ALIAS())) {
                init = WeightInit.RELU_UNIFORM;
            } else if (kerasInit.equals(conf.getINIT_ONE()) ||
                    kerasInit.equals(conf.getINIT_ONES()) ||
                    kerasInit.equals(conf.getINIT_ONES_ALIAS())) {
                init = WeightInit.ONES;
            } else if (kerasInit.equals(conf.getINIT_ZERO()) ||
                    kerasInit.equals(conf.getINIT_ZEROS()) ||
                    kerasInit.equals(conf.getINIT_ZEROS_ALIAS())) {
                init = WeightInit.ZERO;
            } else if (kerasInit.equals(conf.getINIT_UNIFORM()) ||
                    kerasInit.equals(conf.getINIT_RANDOM_UNIFORM()) ||
                    kerasInit.equals(conf.getINIT_RANDOM_UNIFORM_ALIAS())) {
                if (kerasMajorVersion == 2) {
                    double minVal = (double) initConfig.get(conf.getLAYER_FIELD_INIT_MINVAL());
                    double maxVal = (double) initConfig.get(conf.getLAYER_FIELD_INIT_MAXVAL());
                    dist = new UniformDistribution(minVal, maxVal);
                } else {
                    double scale = 0.05;
                    if (initConfig.containsKey(conf.getLAYER_FIELD_INIT_SCALE()))
                        scale = (double) initConfig.get(conf.getLAYER_FIELD_INIT_SCALE());
                    dist = new UniformDistribution(-scale, scale);
                }
                init = WeightInit.DISTRIBUTION;
            } else if (kerasInit.equals(conf.getINIT_NORMAL()) ||
                    kerasInit.equals(conf.getINIT_RANDOM_NORMAL()) ||
                    kerasInit.equals(conf.getINIT_RANDOM_NORMAL_ALIAS())) {
                if (kerasMajorVersion == 2) {
                    double mean = (double) initConfig.get(conf.getLAYER_FIELD_INIT_MEAN());
                    double stdDev = (double) initConfig.get(conf.getLAYER_FIELD_INIT_STDDEV());
                    dist = new NormalDistribution(mean, stdDev);
                } else {
                    double scale = 0.05;
                    if (initConfig.containsKey(conf.getLAYER_FIELD_INIT_SCALE()))
                        scale = (double) initConfig.get(conf.getLAYER_FIELD_INIT_SCALE());
                    dist = new NormalDistribution(0, scale);
                }
                init = WeightInit.DISTRIBUTION;
            } else if (kerasInit.equals(conf.getINIT_CONSTANT()) ||
                    kerasInit.equals(conf.getINIT_CONSTANT_ALIAS())) {
                double value = (double) initConfig.get(conf.getLAYER_FIELD_INIT_VALUE());
                dist = new ConstantDistribution(value);
                init = WeightInit.DISTRIBUTION;
            } else if (kerasInit.equals(conf.getINIT_ORTHOGONAL()) ||
                    kerasInit.equals(conf.getINIT_ORTHOGONAL_ALIAS())) {
                if (kerasMajorVersion == 2) {
                    double gain;
                    try {
                        gain = (double) initConfig.get(conf.getLAYER_FIELD_INIT_GAIN());
                    } catch (Exception e) {
                        gain = (int) initConfig.get(conf.getLAYER_FIELD_INIT_GAIN());
                    }
                    dist = new OrthogonalDistribution(gain);
                } else {
                    double scale = 1.1;
                    if (initConfig.containsKey(conf.getLAYER_FIELD_INIT_SCALE()))
                        scale = (double) initConfig.get(conf.getLAYER_FIELD_INIT_SCALE());
                    dist = new OrthogonalDistribution(scale);
                }
                init = WeightInit.DISTRIBUTION;
            } else if (kerasInit.equals(conf.getINIT_TRUNCATED_NORMAL()) ||
                    kerasInit.equals(conf.getINIT_TRUNCATED_NORMAL_ALIAS())) {
                double mean = (double) initConfig.get(conf.getLAYER_FIELD_INIT_MEAN());
                double stdDev = (double) initConfig.get(conf.getLAYER_FIELD_INIT_STDDEV());
                dist = new TruncatedNormalDistribution(mean, stdDev);
                init = WeightInit.DISTRIBUTION;
            } else if (kerasInit.equals(conf.getINIT_IDENTITY()) ||
                    kerasInit.equals(conf.getINIT_IDENTITY_ALIAS())) {
                if (kerasMajorVersion == 2) {
                    double gain = (double) initConfig.get(conf.getLAYER_FIELD_INIT_GAIN());
                    if (gain != 1.)
                        log.warn("Scaled identity weight init not supported, setting gain=1");
                } else {
                    double scale = 1.;
                    if (initConfig.containsKey(conf.getLAYER_FIELD_INIT_SCALE()))
                        scale = (double) initConfig.get(conf.getLAYER_FIELD_INIT_SCALE());
                    if (scale != 1.)
                        log.warn("Scaled identity weight init not supported, setting scale=1");
                }
                init = WeightInit.IDENTITY;
            } else if (kerasInit.equals(conf.getINIT_VARIANCE_SCALING())) {
                double scale;
                try {
                    scale = (double) initConfig.get(conf.getLAYER_FIELD_INIT_SCALE());
                } catch (Exception e) {
                    scale = (int) initConfig.get(conf.getLAYER_FIELD_INIT_SCALE());
                }
                if (scale != 1.)
                    log.warn("Scaled identity weight init not supported, setting scale=1");
                String mode = (String) initConfig.get(conf.getLAYER_FIELD_INIT_MODE());
                String distribution = (String) initConfig.get(conf.getLAYER_FIELD_INIT_DISTRIBUTION());
                switch (mode) {
                    case "fan_in":
                        if (distribution.equals("normal")) {
                            init = WeightInit.VAR_SCALING_NORMAL_FAN_IN;
                        } else {
                            init = WeightInit.VAR_SCALING_UNIFORM_FAN_IN;
                        }
                        break;
                    case "fan_out":
                        if (distribution.equals("normal")) {
                            init = WeightInit.VAR_SCALING_NORMAL_FAN_OUT;
                        } else {
                            init = WeightInit.VAR_SCALING_UNIFORM_FAN_OUT;
                        }
                        break;
                    case "fan_avg":
                        if (distribution.equals("normal")) {
                            init = WeightInit.VAR_SCALING_NORMAL_FAN_AVG;
                        } else {
                            init = WeightInit.VAR_SCALING_UNIFORM_FAN_AVG;
                        }
                        break;
                    default:
                        throw new InvalidKerasConfigurationException("Initialization argument 'mode' has to be either " +
                                "fan_in, fan_out or fan_avg");
                }
            } else {
                throw new UnsupportedKerasConfigurationException("Unknown keras weight initializer " + kerasInit);
            }
        }
        return new Pair<>(init, dist);
    }
1032272
def add_weight(cls):
  """"""
  @functools.wraps(cls.add_weight)
  def _add_weight(self,
                  name=None,
                  shape=None,
                  dtype=None,
                  initializer=None,
                  regularizer=None,
                  **kwargs):
    """Adds weight."""
    if isinstance(initializer, tf.keras.layers.Layer):
      weight = initializer(shape, dtype)
      self._trainable_weights.extend(initializer.trainable_weights)  # pylint: disable=protected-access
      self._non_trainable_weights.extend(initializer.non_trainable_weights)  # pylint: disable=protected-access
      if regularizer is not None:
        # TODO(trandustin): Replace need for this with
        # Layer._handle_weight_regularization. For Eager compatibility, random
        # variable __init__s cannot apply TF ops (cl/220898007).
        def loss_fn():
          """Creates a regularization loss `Tensor`."""
          with tf.name_scope(name + '/Regularizer'):
            return regularizer(initializer(shape, dtype))
        self.add_loss(loss_fn)
      return weight
    return super(cls, self).add_weight(name=name,
                                       shape=shape,
                                       dtype=dtype,
                                       initializer=initializer,
                                       regularizer=regularizer,
                                       **kwargs)
  cls.add_weight = _add_weight
  return cls
865827
def convert_dense(builder, layer, input_names, output_names, keras_layer):
    
    # Get input and output names
    input_name, output_name = (input_names[0], output_names[0])

    has_bias = keras_layer.bias
    # Get the weights from keras
    W = keras_layer.get_weights ()[0].T
    Wb = keras_layer.get_weights ()[1].T if has_bias else None

    builder.add_inner_product(name = layer,
            W = W,
            b = Wb,
            input_channels = keras_layer.input_dim,
            output_channels = keras_layer.output_dim,
            has_bias = has_bias,
            input_name = input_name,
            output_name = output_name)
476965
@Override
    public InputType getOutputType(InputType... inputType) throws InvalidKerasConfigurationException {
        if (inputType.length > 1)
            throw new InvalidKerasConfigurationException(
                    "Keras Upsampling layer accepts only one input (received " + inputType.length + ")");
        return this.getUpsampling2DLayer().getOutputType(-1, inputType[0]);
    }
67748
private double getMomentumFromConfig(Map<String, Object> layerConfig) throws InvalidKerasConfigurationException {
        Map<String, Object> innerConfig = KerasLayerUtils.getInnerLayerConfigFromConfig(layerConfig, conf);
        if (!innerConfig.containsKey(LAYER_FIELD_MOMENTUM))
            throw new InvalidKerasConfigurationException(
                    "Keras BatchNorm layer config missing " + LAYER_FIELD_MOMENTUM + " field");
        return (double) innerConfig.get(LAYER_FIELD_MOMENTUM);
    }
510235
def init_dense_weight(layer):
    '''
    '''
    units = layer.units
    weight = np.eye(units)
    bias = np.zeros(units)
    layer.set_weights(
        (add_noise(weight, np.array([0, 1])), add_noise(bias, np.array([0, 1])))
    )
448620
def initialize(self):
        
        self.randomize()
        self.dweight = Numeric.zeros((self.fromLayer.size, \
                                      self.toLayer.size), 'f')
        self.wed = Numeric.zeros((self.fromLayer.size, \
                                  self.toLayer.size), 'f')
        self.wedLast = Numeric.zeros((self.fromLayer.size, \
                                      self.toLayer.size), 'f')
153629
public static void removeDefaultWeights(Map<String, INDArray> weights, KerasLayerConfiguration conf) {
        if (weights.size() > 2) {
            Set<String> paramNames = weights.keySet();
            paramNames.remove(conf.getKERAS_PARAM_NAME_W());
            paramNames.remove(conf.getKERAS_PARAM_NAME_B());
            String unknownParamNames = paramNames.toString();
            log.warn("Attemping to set weights for unknown parameters: "
                    + unknownParamNames.substring(1, unknownParamNames.length() - 1));
        }
    }
1575
@Override
    public InputType getOutputType(InputType... inputType) throws InvalidKerasConfigurationException {
        if (inputType.length > 1)
            throw new InvalidKerasConfigurationException(
                            "Keras ZeroPadding layer accepts only one input (received " + inputType.length + ")");
        return this.getZeroPadding2DLayer().getOutputType(-1, inputType[0]);
    }
987323
@Override
    public void setWeights(Map<String, INDArray> weights) throws InvalidKerasConfigurationException {
        this.weights = new HashMap<>();
        if (weights.containsKey(conf.getKERAS_PARAM_NAME_W())) {
            INDArray kerasParamValue = weights.get(conf.getKERAS_PARAM_NAME_W());
            INDArray paramValue;
            switch (this.getDimOrder()) {
                case TENSORFLOW:
                    paramValue = kerasParamValue.permute(2, 1, 0);
                    paramValue = paramValue.reshape(
                            paramValue.size(0), paramValue.size(1),
                            paramValue.size(2), 1);
                    break;

                case THEANO:
                    paramValue = kerasParamValue.permute(2, 1, 0);
                    paramValue = paramValue.reshape(
                            paramValue.size(0), paramValue.size(1),
                            paramValue.size(2), 1).dup();
                    for (int i = 0; i < paramValue.tensorsAlongDimension(2, 3); i++) {
                        INDArray copyFilter = paramValue.tensorAlongDimension(i, 2, 3).dup();
                        double[] flattenedFilter = copyFilter.ravel().data().asDouble();
                        ArrayUtils.reverse(flattenedFilter);
                        INDArray newFilter = Nd4j.create(flattenedFilter, copyFilter.shape());
                        INDArray inPlaceFilter = paramValue.tensorAlongDimension(i, 2, 3);
                        inPlaceFilter.muli(0).addi(newFilter);
                    }
                    break;
                default:
                    throw new InvalidKerasConfigurationException("Unknown keras backend " + this.getDimOrder());
            }

            this.weights.put(ConvolutionParamInitializer.WEIGHT_KEY, paramValue);
        } else
            throw new InvalidKerasConfigurationException(
                    "Parameter " + conf.getKERAS_PARAM_NAME_W() + " does not exist in weights");

        if (hasBias) {
            if (weights.containsKey(conf.getKERAS_PARAM_NAME_B()))
                this.weights.put(ConvolutionParamInitializer.BIAS_KEY, weights.get(conf.getKERAS_PARAM_NAME_B()));
            else
                throw new InvalidKerasConfigurationException(
                        "Parameter " + conf.getKERAS_PARAM_NAME_B() + " does not exist in weights");
        }
        removeDefaultWeights(weights, conf);
    }
584170
@Override
    public InputType getOutputType(InputType... inputType) throws InvalidKerasConfigurationException,
    UnsupportedKerasConfigurationException {
        if (inputType.length > 1)
            throw new InvalidKerasConfigurationException(
                    "Keras Loss layer accepts only one input (received " + inputType.length + ")");
        return this.getLossLayer(inputType[0]).getOutputType(-1, inputType[0]);
    }
883691
public MultiLayerConfiguration getMultiLayerConfiguration()
            throws InvalidKerasConfigurationException, UnsupportedKerasConfigurationException {
        if (!this.className.equals(config.getFieldClassNameSequential()))
            throw new InvalidKerasConfigurationException(
                    "Keras model class name " + this.className + " incompatible with MultiLayerNetwork");
        if (this.inputLayerNames.size() != 1)
            throw new InvalidKerasConfigurationException(
                    "MultiLayerNetwork expects only 1 input (found " + this.inputLayerNames.size() + ")");
        if (this.outputLayerNames.size() != 1)
            throw new InvalidKerasConfigurationException(
                    "MultiLayerNetwork expects only 1 output (found " + this.outputLayerNames.size() + ")");

        NeuralNetConfiguration.Builder modelBuilder = new NeuralNetConfiguration.Builder();

        if (optimizer != null) {
            modelBuilder.updater(optimizer);
        }

        NeuralNetConfiguration.ListBuilder listBuilder = modelBuilder.list();

        /* Add layers one at a time. */
        KerasLayer prevLayer = null;
        int layerIndex = 0;
        for (KerasLayer layer : this.layersOrdered) {
            if (layer.isLayer()) {
                int nbInbound = layer.getInboundLayerNames().size();
                if (nbInbound != 1)
                    throw new InvalidKerasConfigurationException(
                            "Layers in MultiLayerConfiguration must have exactly one inbound layer (found "
                                    + nbInbound + " for layer " + layer.getLayerName() + ")");
                if (prevLayer != null) {
                    InputType[] inputTypes = new InputType[1];
                    InputPreProcessor preprocessor;
                    if (prevLayer.isInputPreProcessor()) {
                        inputTypes[0] = this.outputTypes.get(prevLayer.getInboundLayerNames().get(0));
                        preprocessor = prevLayer.getInputPreprocessor(inputTypes);
                    } else {
                        inputTypes[0] = this.outputTypes.get(prevLayer.getLayerName());
                        preprocessor = layer.getInputPreprocessor(inputTypes);
                    }
                    if (preprocessor != null)
                        listBuilder.inputPreProcessor(layerIndex, preprocessor);
                }
                listBuilder.layer(layerIndex++, layer.getLayer());
            } else if (layer.getVertex() != null)
                throw new InvalidKerasConfigurationException("Cannot add vertex to MultiLayerConfiguration (class name "
                        + layer.getClassName() + ", layer name " + layer.getLayerName() + ")");
            prevLayer = layer;
        }

        InputType inputType = this.layersOrdered.get(0).getOutputType();
        if (inputType != null)
            listBuilder.setInputType(inputType);

        /* Whether to use standard backprop (or BPTT) or truncated BPTT. */
        if (this.useTruncatedBPTT && this.truncatedBPTT > 0)
            listBuilder.backpropType(BackpropType.TruncatedBPTT).tBPTTForwardLength(truncatedBPTT)
                    .tBPTTBackwardLength(truncatedBPTT);
        else
            listBuilder.backpropType(BackpropType.Standard);
        return listBuilder.build();
    }
1137394
public InputType getOutputType(InputType... inputType) throws InvalidKerasConfigurationException {
        if (inputType.length > 1)
            throw new InvalidKerasConfigurationException(
                    "Keras PReLU layer accepts only one input (received " + inputType.length + ")");
        InputType inType = inputType[0];

        // Dynamically infer input shape of PReLU layer from input type
        PReLULayer shapedLayer = (PReLULayer) this.layer;
        shapedLayer.setInputShape(inType.getShape());
        this.layer = shapedLayer;

        return this.getPReLULayer().getOutputType(-1, inputType[0]);
    }
852948
public static Initializer defaultInitializer() {
      return newInitializer()
              .setCloseConcurrency(0)
              .setTestOnLogicalOpen(false)
              .setTestOnLogicalClose(false)
              .setIncompleteTransactionOnClosePolicy(ConnectionPoolConnection.IncompleteTransactionPolicy.REPORT)
              .setOpenStatementOnClosePolicy(ConnectionPoolConnection.OpenStatementPolicy.REPORT)
              .setForceRealClosePolicy(ConnectionPoolConnection.ForceRealClosePolicy.CONNECTION_WITH_LIMIT)
              .setActivityTimeoutPolicy(ConnectionPoolConnection.ActivityTimeoutPolicy.LOG)
              .setCloseTimeLimitMillis(10 * 1000L)
              .setActiveTimeout(60, TimeUnit.SECONDS)
              .setConnectionLifetime(15, TimeUnit.MINUTES)
              .setMaxConcurrentReconnects(2)
              .setMaxReconnectDelay(1, TimeUnit.MINUTES)
              .setActiveTimeoutMonitorFrequency(30, TimeUnit.SECONDS);
   }
28107
public static <Q> LockingLazyVar<Q> make( final LazyVarInit<Q> init ) {
    return new LockingLazyVar<Q>(){
      protected Q init()
      {
        return init.init();
      }
    };
  }
633820
@Nonnull
    public static <N, E> SSSPResult<N, E> findSSSP(Graph<N, E> graph, N init, EdgeWeights<E> edgeWeights) {
        DijkstraSSSP<N, E> dijkstra = new DijkstraSSSP<>(graph, init, edgeWeights);
        dijkstra.findSSSP();
        return dijkstra;
    }
978067
private int getInputDimFromConfig(Map<String, Object> layerConfig) throws InvalidKerasConfigurationException {
        Map<String, Object> innerConfig = KerasLayerUtils.getInnerLayerConfigFromConfig(layerConfig, conf);
        if (!innerConfig.containsKey(conf.getLAYER_FIELD_INPUT_DIM()))
            throw new InvalidKerasConfigurationException(
                    "Keras Embedding layer config missing " + conf.getLAYER_FIELD_INPUT_DIM() + " field");
        return (int) innerConfig.get(conf.getLAYER_FIELD_INPUT_DIM());
    }