map of spark models to input-output tuples
    Each lambda gets the corresponding input or output column name from the model
481969
def build_io_name_map():
    '''
    
    '''
    map = {
        "pyspark.ml.feature.BucketedRandomProjectionLSHModel": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.regression.AFTSurvivalRegressionModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol")]
        ),
        "pyspark.ml.feature.ElementwiseProduct": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.MinHashLSHModel": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.Word2VecModel": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.IndexToString": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.ChiSqSelectorModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.classification.OneVsRestModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol")]
        ),
        "pyspark.ml.regression.GBTRegressionModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol")]
        ),
        "pyspark.ml.classification.GBTClassificationModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol"), 'probability']
        ),
        "pyspark.ml.feature.DCT": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.PCAModel": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.PolynomialExpansion": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.Tokenizer": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.classification.NaiveBayesModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol"), model.getOrDefault("probabilityCol")]
        ),
        "pyspark.ml.feature.VectorSlicer": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.StopWordsRemover": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.NGram": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.Bucketizer": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.regression.RandomForestRegressionModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol")]
        ),
        "pyspark.ml.classification.RandomForestClassificationModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol"), model.getOrDefault("probabilityCol")]
        ),
        "pyspark.ml.regression.DecisionTreeRegressionModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol")]
        ),
        "pyspark.ml.classification.DecisionTreeClassificationModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol"), model.getOrDefault("probabilityCol")]
        ),
        "pyspark.ml.feature.VectorIndexerModel": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.regression.GeneralizedLinearRegressionModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol")]
        ),
        "pyspark.ml.regression.LinearRegressionModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol")]
        ),
        "pyspark.ml.feature.ImputerModel": (
            lambda model: model.getOrDefault("inputCols"),
            lambda model: model.getOrDefault("outputCols")
        ),
        "pyspark.ml.feature.MaxAbsScalerModel": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.MinMaxScalerModel": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.StandardScalerModel": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.Normalizer": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.Binarizer": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.classification.LinearSVCModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol")]
        ),
        "pyspark.ml.classification.LogisticRegressionModel": (
            lambda model: [model.getOrDefault("featuresCol")],
            lambda model: [model.getOrDefault("predictionCol"), model.getOrDefault("probabilityCol")]
        ),
        "pyspark.ml.feature.OneHotEncoderModel": (
            lambda model: model.getOrDefault("inputCols"),
            lambda model: model.getOrDefault("outputCols")
        ),
        "pyspark.ml.feature.StringIndexerModel": (
            lambda model: [model.getOrDefault("inputCol")],
            lambda model: [model.getOrDefault("outputCol")]
        ),
        "pyspark.ml.feature.VectorAssembler": (
            lambda model: model.getOrDefault("inputCols"),
            lambda model: [model.getOrDefault("outputCol")]
        )
    }
    return map
81986
def columns(self, model=None):
        
        column = self.column(model=model)
        if column:
            yield column

        check = self.__value
        if not isinstance(check, (list, set, tuple)):
            check = (check,)

        for val in check:
            if isinstance(val, (Query, QueryCompound)):
                for col in val.columns(model):
                    yield col
1068636
def outputAlphabet(self):
        
        return set(
            chain.from_iterable(
                outputSymbols for
                (inState, inputSymbol, outState, outputSymbols)
                in self._transitions
            )
        )
383672
def _to_java(self):
        
        sc = SparkContext._active_spark_context
        java_models = [model._to_java() for model in self.models]
        java_models_array = JavaWrapper._new_java_array(
            java_models, sc._gateway.jvm.org.apache.spark.ml.classification.ClassificationModel)
        metadata = JavaParams._new_java_obj("org.apache.spark.sql.types.Metadata")
        _java_obj = JavaParams._new_java_obj("org.apache.spark.ml.classification.OneVsRestModel",
                                             self.uid, metadata.empty(), java_models_array)
        _java_obj.set("classifier", self.getClassifier()._to_java())
        _java_obj.set("featuresCol", self.getFeaturesCol())
        _java_obj.set("labelCol", self.getLabelCol())
        _java_obj.set("predictionCol", self.getPredictionCol())
        return _java_obj
803777
public List<MutableInputColumn<?>> getOutputColumns() {
        final Component component = getComponentInstanceForQuestioning();
        if (component == null) {
            // as long as the transformer is not configured, just return an
            // empty list
            return Collections.emptyList();
        }

        final Transformer transformer = (Transformer) component;
        
        final OutputColumns outputColumns;
        try {
            outputColumns = transformer.getOutputColumns();
        } catch (final Exception e) {
            logger.error("getOutputColumn() threw unexpected exception on transformer: " + transformer, e);
            return Collections.emptyList();
        }
        if (outputColumns == null) {
            logger.error("getOutputColumns() returned null on transformer: " + transformer);
            return Collections.emptyList();
        }
        boolean changed = false;

        // adjust the amount of output columns
        final int expectedCols = outputColumns.getColumnCount();
        final int existingCols = _outputColumns.size();
        if (expectedCols != existingCols) {
            changed = true;
            final int colDiff = expectedCols - existingCols;
            if (colDiff > 0) {
                for (int i = 0; i < colDiff; i++) {
                    final int nextIndex = _outputColumns.size();
                    final String name = getColumnName(outputColumns, nextIndex);
                    final String id = _id + "-" + _idGenerator.nextId();
                    final TransformedInputColumn<Object> column = new TransformedInputColumn<>(name, id);
                    /*
                     * If the transformer has the annotation
                     * "HideOutputColumns", the output columns are hidden in
                     * other components as input columns. If the user wants to
                     * use them, he can select them in the transformer.
                     */
                    final HideOutputColumns hideOutputColumnsAnnotation =
                            this.getDescriptor().getAnnotation(HideOutputColumns.class);
                    if (hideOutputColumnsAnnotation != null && hideOutputColumnsAnnotation.isHidden()) {
                        column.setHidden(true);
                    }

                    _outputColumns.add(column);
                    _automaticOutputColumnNames.add(name);
                }
            } else if (colDiff < 0) {
                for (int i = 0; i < Math.abs(colDiff); i++) {
                    // remove from the tail
                    _outputColumns.remove(_outputColumns.size() - 1);
                    _automaticOutputColumnNames.remove(_automaticOutputColumnNames.size() - 1);
                }
            }

            // reset the names when the number of output columns change and the
            // initial name has changed
            for (int i = 0; i < expectedCols; i++) {
                final MutableInputColumn<?> column = _outputColumns.get(i);
                final String previousProposedName = column.getInitialName();
                final String newProposedName = outputColumns.getColumnName(i);
                if (newProposedName != null && !newProposedName.equals(previousProposedName)) {
                    column.setName(newProposedName);
                }
            }
        }

        // automatically update names and types of columns if they have not been
        // manually set
        for (int i = 0; i < expectedCols; i++) {
            final String proposedName = getColumnName(outputColumns, i);
            final Class<?> dataType = outputColumns.getColumnType(i);

            final TransformedInputColumn<?> col = (TransformedInputColumn<?>) _outputColumns.get(i);
            col.setInitialName(proposedName);
            if (dataType != col.getDataType()) {
                col.setDataType(dataType);
                changed = true;
            }

            final String automaticName = _automaticOutputColumnNames.get(i);
            final String columnName = col.getName();
            if (StringUtils.isNullOrEmpty(columnName) || automaticName.equals(columnName)) {
                if (proposedName != null) {
                    col.setName(proposedName);
                    _automaticOutputColumnNames.set(i, proposedName);
                }
            }
        }

        if (changed) {
            // notify listeners
            onOutputChanged();
        }

        return Collections.unmodifiableList(_outputColumns);
    }
58967
def build_model_output(cls, name='output'):
        
        return cls(name, PortDirection.OUTPUT, type=PortType.MODEL)
732946
public List<MutableInputColumn<?>> getOutputColumns() {
        if (!isConfigured()) {
            // as long as the transformer is not configured, just return an
            // empty list
            return Collections.emptyList();
        }

        final Transformer<?> component = getComponentInstance();
        final TransformerBeanDescriptor<T> descriptor = getDescriptor();

        final InjectionManager injectionManager = getAnalysisJobBuilder().getConfiguration().getInjectionManager(null);
        final LifeCycleHelper lifeCycleHelper = new LifeCycleHelper(injectionManager, null, false);

        // mimic the configuration of a real transformer bean instance
        final BeanConfiguration beanConfiguration = new ImmutableBeanConfiguration(getConfiguredProperties());
        lifeCycleHelper.assignConfiguredProperties(descriptor, component, beanConfiguration);
        lifeCycleHelper.assignProvidedProperties(descriptor, component);

        // only validate, don't initialize
        lifeCycleHelper.validate(descriptor, component);

        final OutputColumns outputColumns = component.getOutputColumns();
        if (outputColumns == null) {
            throw new IllegalStateException("getOutputColumns() returned null on transformer: " + component);
        }
        boolean changed = false;

        // adjust the amount of output columns
        final int expectedCols = outputColumns.getColumnCount();
        final int existingCols = _outputColumns.size();
        if (expectedCols != existingCols) {
            changed = true;
            int colDiff = expectedCols - existingCols;
            if (colDiff > 0) {
                for (int i = 0; i < colDiff; i++) {
                    final int nextIndex = _outputColumns.size();
                    final String name = getColumnName(outputColumns, nextIndex);
                    final String id = _id + "-" + _idGenerator.nextId();
                    _outputColumns.add(new TransformedInputColumn<Object>(name, id));
                    _automaticOutputColumnNames.add(name);
                }
            } else if (colDiff < 0) {
                for (int i = 0; i < Math.abs(colDiff); i++) {
                    // remove from the tail
                    _outputColumns.remove(_outputColumns.size() - 1);
                    _automaticOutputColumnNames.remove(_automaticOutputColumnNames.size() - 1);
                }
            }

            // reset the names when the number of output columns change and the
            // initial name has changed
            for (int i = 0; i < expectedCols; i++) {
                final MutableInputColumn<?> column = _outputColumns.get(i);
                final String previousProposedName = column.getInitialName();
                final String newProposedName = outputColumns.getColumnName(i);
                if (newProposedName != null && !newProposedName.equals(previousProposedName)) {
                    column.setName(newProposedName);
                }
            }
        }

        // automatically update names and types of columns if they have not been
        // manually set
        for (int i = 0; i < expectedCols; i++) {
            final String proposedName = getColumnName(outputColumns, i);
            Class<?> dataType = outputColumns.getColumnType(i);
            if (dataType == null) {
                dataType = descriptor.getOutputDataType();
            }

            final TransformedInputColumn<?> col = (TransformedInputColumn<?>) _outputColumns.get(i);
            col.setInitialName(proposedName);
            if (dataType != col.getDataType()) {
                col.setDataType(dataType);
                changed = true;
            }

            final String automaticName = _automaticOutputColumnNames.get(i);
            final String columnName = col.getName();
            if (StringUtils.isNullOrEmpty(columnName) || automaticName.equals(columnName)) {
                if (proposedName != null) {
                    col.setName(proposedName);
                    _automaticOutputColumnNames.set(i, proposedName);
                }
            }
        }

        if (changed) {
            // notify listeners
            onOutputChanged();
        }

        return Collections.unmodifiableList(_outputColumns);
    }
1278384
def to_java_rdd(jsc, features, labels, batch_size):
    
    data_sets = java_classes.ArrayList()
    num_batches = int(len(features) / batch_size)
    for i in range(num_batches):
        xi = ndarray(features[:batch_size].copy())
        yi = ndarray(labels[:batch_size].copy())
        data_set = java_classes.DataSet(xi.array, yi.array)
        data_sets.add(data_set)
        features = features[batch_size:]
        labels = labels[batch_size:]

    return jsc.parallelize(data_sets)
333007
def get_input_shape(sym, proto_obj):
    """"""
    arg_params = proto_obj.arg_dict
    aux_params = proto_obj.aux_dict

    model_input_shape = [data[1] for data  in proto_obj.model_metadata.get('input_tensor_data')]
    data_names = [data[0] for data  in proto_obj.model_metadata.get('input_tensor_data')]

    # creating dummy inputs
    inputs = []
    for  in_shape in model_input_shape:
        inputs.append(nd.ones(shape=in_shape))

    data_shapes = []
    for idx, input_name in enumerate(data_names):
        data_shapes.append((input_name, inputs[idx].shape))

    ctx = context.cpu()
    # create a module
    mod = module.Module(symbol=sym, data_names=data_names, context=ctx, label_names=None)
    mod.bind(for_training=False, data_shapes=data_shapes, label_shapes=None)
    mod.set_params(arg_params=arg_params, aux_params=aux_params)

    data_forward = []
    for idx, input_name in enumerate(data_names):
        val = inputs[idx]
        data_forward.append(val)

    mod.forward(io.DataBatch(data_forward))
    result = mod.get_outputs()[0].asnumpy()

    return result.shape
496229
def _transform(self, df):
        
        output_col = self.getOutputCol()
        label_col = self.getLabelCol()
        new_schema = copy.deepcopy(df.schema)
        new_schema.add(StructField(output_col, StringType(), True))

        rdd = df.rdd.coalesce(1)
        features = np.asarray(
            rdd.map(lambda x: from_vector(x.features)).collect())
        # Note that we collect, since executing this on the rdd would require model serialization once again
        model = model_from_yaml(self.get_keras_model_config())
        model.set_weights(self.weights.value)
        predictions = rdd.ctx.parallelize(
            model.predict_classes(features)).coalesce(1)
        predictions = predictions.map(lambda x: tuple(str(x)))

        results_rdd = rdd.zip(predictions).map(lambda x: x[0] + x[1])
        results_df = df.sql_ctx.createDataFrame(results_rdd, new_schema)
        results_df = results_df.withColumn(
            output_col, results_df[output_col].cast(DoubleType()))
        results_df = results_df.withColumn(
            label_col, results_df[label_col].cast(DoubleType()))

        return results_df
593876
def make_mapper(features):
    
    if not features:
        features = Feature(input=[], transformer=NullTransformer())
    if not iterable(features):
        features = (features, )
    return DataFrameMapper(
        [t.as_input_transformer_tuple() for t in features],
        input_df=True)
758181
def _get_stats_column_names(cls):
        
        columns = []
        stats = ('num_bytes', 'num_rows', 'location', 'partitioning_type',
                 'clustering_fields')
        stat_components = ('label', 'value', 'description', 'include')
        for stat_id in stats:
            for stat_component in stat_components:
                columns.append('stats:{}:{}'.format(stat_id, stat_component))
        return tuple(columns)
285501
def from_arrays_of_labels(cls, arrayOfLabels, inputCols, outputCols=None,
                              handleInvalid=None):
        
        sc = SparkContext._active_spark_context
        java_class = sc._gateway.jvm.java.lang.String
        jlabels = StringIndexerModel._new_java_array(arrayOfLabels, java_class)
        model = StringIndexerModel._create_from_java_class(
            "org.apache.spark.ml.feature.StringIndexerModel", jlabels)
        model.setInputCols(inputCols)
        if outputCols is not None:
            model.setOutputCols(outputCols)
        if handleInvalid is not None:
            model.setHandleInvalid(handleInvalid)
        return model
648298
def map_column(self, keys, func):
        
        return [[func(v) for v in self[key]] for key in keys]
546750
function mapInputType(model) {
    const modelName = `${model.modelName}Input`;

    types[modelName] = {
        generated: false,
        name: modelName,
        meta: {
            category: 'TYPE',
            input: true,
            fields: {}
        }
    };

    _.forEach(model.definition.properties, (property, key) => {
        mapProperty(model, property, modelName, key, true);
    });
}
1330131
@Override
    public List<Values> mapToValues(String rowKey, Map<String, String> columns, Tuple input) {
        List<Values> values = new ArrayList<Values>();
        Set<String> vals = columns.keySet();
        for(String columnName : vals){
            if(this.isDrpc){
                values.add(new Values(input.getValue(0), rowKey, columnName));
            } else {
                values.add(new Values(rowKey, columnName));
            }
        }
        return values;
    }
972518
@Override
    protected void writeToOutput(final Map<String, Object> row)
            throws IOException {
        if (generator == null) {
            ObjectMapper mapper = new ObjectMapper();
            BsonFactory factory = new BsonFactory(mapper);
            factory.enable(BsonGenerator.Feature.ENABLE_STREAMING);
            generator = factory.createJsonGenerator(getOutputStream());
            generator.writeStartArray(); // [
        }

        // Convert the tuple into a map
        generator.writeObject(row);
    }