Normalize by zero mean unit variance

@param schema the schema to use
to create the data frame
@param data   the data to normalize
@return a zero mean unit variance centered
rdd
431914
public static JavaRDD<List<Writable>> zeromeanUnitVariance(Schema schema, JavaRDD<List<Writable>> data) {
        return zeromeanUnitVariance(schema, data, Collections.<String>emptyList());
    }
1163116
def _inferSchema(self, rdd, samplingRatio=None, names=None):
        
        first = rdd.first()
        if not first:
            raise ValueError("The first row in RDD is empty, "
                             "can not infer schema")
        if type(first) is dict:
            warnings.warn("Using RDD of dict to inferSchema is deprecated. "
                          "Use pyspark.sql.Row instead")

        if samplingRatio is None:
            schema = _infer_schema(first, names=names)
            if _has_nulltype(schema):
                for row in rdd.take(100)[1:]:
                    schema = _merge_type(schema, _infer_schema(row, names=names))
                    if not _has_nulltype(schema):
                        break
                else:
                    raise ValueError("Some of types cannot be determined by the "
                                     "first 100 rows, please try again with sampling")
        else:
            if samplingRatio < 0.99:
                rdd = rdd.sample(False, float(samplingRatio))
            schema = rdd.map(lambda row: _infer_schema(row, names)).reduce(_merge_type)
        return schema
1316449
public static DataAnalysis analyze(Schema schema, RecordReader rr) {
        return analyze(schema, rr, DEFAULT_MAX_HISTOGRAM_BUCKETS);
    }
1145109
def df_to_simple_rdd(df, categorical=False, nb_classes=None, features_col='features', label_col='label'):
    
    sql_context = df.sql_ctx
    sql_context.registerDataFrameAsTable(df, "temp_table")
    selected_df = sql_context.sql(
        "SELECT {0} AS features, {1} as label from temp_table".format(features_col, label_col))
    if isinstance(selected_df.first().features, MLLibVector):
        lp_rdd = selected_df.rdd.map(
            lambda row: LabeledPoint(row.label, row.features))
    else:
        lp_rdd = selected_df.rdd.map(lambda row: LabeledPoint(
            row.label, MLLibVectors.fromML(row.features)))
    rdd = lp_to_simple_rdd(lp_rdd, categorical, nb_classes)
    return rdd
1154075
public Object normalize(Map<String, Object> dataset) throws JsonLdError {
        // create quads and map bnodes to their associated quads
        final List<Object> quads = new ArrayList<Object>();
        final Map<String, Object> bnodes = newMap();
        for (String graphName : dataset.keySet()) {
            final List<Map<String, Object>> triples = (List<Map<String, Object>>) dataset
                    .get(graphName);
            if (JsonLdConsts.DEFAULT.equals(graphName)) {
                graphName = null;
            }
            for (final Map<String, Object> quad : triples) {
                if (graphName != null) {
                    if (graphName.indexOf("_:") == 0) {
                        final Map<String, Object> tmp = newMap();
                        tmp.put("type", "blank node");
                        tmp.put("value", graphName);
                        quad.put("name", tmp);
                    } else {
                        final Map<String, Object> tmp = newMap();
                        tmp.put("type", "IRI");
                        tmp.put("value", graphName);
                        quad.put("name", tmp);
                    }
                }
                quads.add(quad);

                final String[] attrs = new String[] { "subject", "object", "name" };
                for (final String attr : attrs) {
                    if (quad.containsKey(attr) && "blank node"
                            .equals(((Map<String, Object>) quad.get(attr)).get("type"))) {
                        final String id = (String) ((Map<String, Object>) quad.get(attr))
                                .get("value");
                        if (!bnodes.containsKey(id)) {
                            bnodes.put(id, new LinkedHashMap<String, List<Object>>() {
                                {
                                    put("quads", new ArrayList<Object>());
                                }
                            });
                        }
                        ((List<Object>) ((Map<String, Object>) bnodes.get(id)).get("quads"))
                                .add(quad);
                    }
                }
            }
        }

        // mapping complete, start canonical naming
        final NormalizeUtils normalizeUtils = new NormalizeUtils(quads, bnodes,
                new UniqueNamer("_:c14n"), opts);
        return normalizeUtils.hashBlankNodes(bnodes.keySet());
    }
913054
public static double geometricMean(FlatDataCollection flatDataCollection) {
	    int n = 0;
        double geometricMean = 0.0;

        Iterator<Double> it = flatDataCollection.iteratorDouble();
        while(it.hasNext()) {
            Double v = it.next();
            if(v != null) {
                if(v <= 0.0) {
                    throw new IllegalArgumentException("Negative or zero values are not allowed.");
                }
                ++n;
                geometricMean+= Math.log(v);
            }
        }
        
        geometricMean= Math.exp(geometricMean/n);
        
        return geometricMean;
    }
229619
@SuppressWarnings("unchecked")
    public CassandraJavaPairRDD<K, V> withAscOrder() {
        CassandraRDD<Tuple2<K, V>> newRDD = rdd().withAscOrder();
        return wrap(newRDD);
    }
366596
public static DataRowsFacade toDataFrame(Schema schema, JavaRDD<List<Writable>> data) {
        JavaSparkContext sc = new JavaSparkContext(data.context());
        SQLContext sqlContext = new SQLContext(sc);
        JavaRDD<Row> rows = data.map(new ToRow(schema));
        return dataRows(sqlContext.createDataFrame(rows, fromSchema(schema)));
    }
875574
public static double[] rowMeans(double[][] data) {
        double[] x = new double[data.length];

        for (int i = 0; i < x.length; i++) {
            x[i] = mean(data[i]);
        }

        return x;
    }
960343
public static <T>  List<FieldVector> toArrowColumnsTimeSeriesHelper(final BufferAllocator bufferAllocator,
                                                                        final Schema schema,
                                                                        List<List<List<T>>> dataVecRecord) {
        //time series length * number of columns
        int numRows = 0;
        for(List<List<T>> timeStep : dataVecRecord) {
            numRows += timeStep.get(0).size() * timeStep.size();
        }

        numRows /= schema.numColumns();


        List<FieldVector> ret = createFieldVectors(bufferAllocator,schema,numRows);
        Map<Integer,Integer> currIndex = new HashMap<>(ret.size());
        for(int i = 0; i < ret.size(); i++) {
            currIndex.put(i,0);
        }
        for(int i = 0; i < dataVecRecord.size(); i++) {
            List<List<T>> record = dataVecRecord.get(i);
            for(int j = 0; j < record.size(); j++) {
                List<T> curr = record.get(j);
                for(int k = 0; k < curr.size(); k++) {
                    Integer idx = currIndex.get(k);
                    FieldVector fieldVector = ret.get(k);
                    T writable = curr.get(k);
                    setValue(schema.getType(k), fieldVector, writable, idx);
                    currIndex.put(k,idx + 1);
                }
            }
        }

        return ret;
    }
1250225
public static ColumnMajorSparseMatrix zero(int rows, int columns, int capacity) {
        return CCSMatrix.zero(rows, columns, capacity);
    }
762381
public static Schema toDatavecSchema(org.apache.arrow.vector.types.pojo.Schema schema) {
        Schema.Builder schemaBuilder = new Schema.Builder();
        for (int i = 0; i < schema.getFields().size(); i++) {
            schemaBuilder.addColumn(metaDataFromField(schema.getFields().get(i)));
        }
        return schemaBuilder.build();
    }
965690
private void dataBufferNormalize() {
        int i = 0;
        int j = 0;
        int end = dataBufferPos;

        // Skip spaces at the start.
        while ((j < end) && (dataBuffer[j] == ' ')) {
            j++;
        }

        // Skip whitespace at the end.
        while ((end > j) && (dataBuffer[end - 1] == ' ')) {
            end--;
        }

        // Start copying to the left.
        while (j < end) {

            char c = dataBuffer[j++];

            // Normalise all other spaces to
            // a single space.
            if (c == ' ') {
                while ((j < end) && (dataBuffer[j++] == ' ')) {
                    continue;
                }
                dataBuffer[i++] = ' ';
                dataBuffer[i++] = dataBuffer[j - 1];
            } else {
                dataBuffer[i++] = c;
            }
        }

        // The new length is <= the old one.
        dataBufferPos = i;
    }
576838
public static JavaRDD<List<Writable>> zeromeanUnitVariance(Schema schema, JavaRDD<List<Writable>> data,
                    List<String> skipColumns) {
        DataRowsFacade frame = DataFrames.toDataFrame(schema, data);
        return DataFrames.toRecords(zeromeanUnitVariance(frame, skipColumns)).getSecond();
    }
1134610
public static SparseMatrix zero(int rows, int columns, int capacity) {
        return CRSMatrix.zero(rows, columns, capacity);
    }
194055
public static Geometry normalize(Geometry geometry) {
        if(geometry == null){
            return null;
        }
        geometry.normalize();
        return geometry;
    }
810033
private void processElementNormal(GeneratorSingleCluster cluster, Node cur) {
    double mean = 0.0;
    double stddev = 1.0;
    String meanstr = ((Element) cur).getAttribute(ATTR_MEAN);
    if(meanstr != null && meanstr.length() > 0) {
      mean = ParseUtil.parseDouble(meanstr);
    }
    String stddevstr = ((Element) cur).getAttribute(ATTR_STDDEV);
    if(stddevstr != null && stddevstr.length() > 0) {
      stddev = ParseUtil.parseDouble(stddevstr);
    }

    // *** New normal distribution generator
    Random random = cluster.getNewRandomGenerator();
    Distribution generator = new NormalDistribution(mean, stddev, random);
    cluster.addGenerator(generator);

    // TODO: check for unknown attributes.
    XMLNodeIterator iter = new XMLNodeIterator(cur.getFirstChild());
    while(iter.hasNext()) {
      Node child = iter.next();
      if(child.getNodeType() == Node.ELEMENT_NODE) {
        LOG.warning("Unknown element in XML specification file: " + child.getNodeName());
      }
    }
  }