Convert numpy array to nd4j array
491890
def _from_numpy(np_array):
    

    # Convert the numpy array to nd4j context dtype
    required_dtype = get_np_dtype(get_context_dtype())
    if np_array.dtype != required_dtype:
        raise Exception("{} is required, got {}".format(
            repr(required_dtype), repr(np_array.dtype)))

    # Nd4j does not have 1-d vectors.
    # So we add a dummy dimension.
    if np_array.ndim == 1:
        np_array = np.expand_dims(np_array, 0)

    # We have to maintain references to all incoming
    # numpy arrays. Else they will get GCed

    # creates a Nd4j array from a numpy array
    # To create an Nd4j array, we need 3 things:
    # buffer, strides, and shape

    # Get the buffer
    # A buffer is basically an array. To get the buffer object
    # we need a pointer to the first element and the size.
    pointer_address, _ = np_array.__array_interface__['data']
    _refs.append(np_array)
    pointer = native_ops.pointerForAddress(pointer_address)
    size = np_array.size
    mapping = {
        np.float64: DoublePointer,
        np.float32: FloatPointer,
    }
    pointer = mapping[required_dtype](pointer)
    buff = Nd4j.createBuffer(pointer, size)
    assert buff.address() == pointer_address
    _refs.append(buff)
    # Get the strides
    # strides = tuple of bytes to step in each
    # dimension when traversing an array.
    elem_size = buff.getElementSize()
    # Make sure word size is same in both python
    # and java worlds
    assert elem_size == np_array.dtype.itemsize
    strides = np_array.strides
    # numpy uses byte wise strides. We have to
    # convert it to word wise strides.
    strides = [dim / elem_size for dim in strides]

    # Finally, shape:
    shape = np_array.shape

    nd4j_array = Nd4j.create(buff, shape, strides, 0)
    assert buff.address() == nd4j_array.data().address()
    return nd4j_array
85828
def write_ndarray(self, result, dst_paths, nodata=None, compress='lzw'):
        """"""

        assert len(dst_paths) == result.shape[2]
        assert result.shape[0] == self._height
        assert result.shape[1] == self._width
        assert result.shape[2] == len(dst_paths)
        with rasterio.open(self._mrio._get_template_for_given_resolution(self._mrio.dst_res, "path")) as src_layer:
            pass # later we need src_layer for src_layer.window_transform(win)
        for i, pth in enumerate(dst_paths):
            dst_path_chunk = self.get_chunk_path_from_layer_path(pth, self.ji)

            result_layer_i = result[:, :, [i]]
            assert result_layer_i.shape[2] == 1
            kwargs = self._mrio._get_template_for_given_resolution(
                res=self._mrio.dst_res, return_="meta").copy()
            kwargs.update({"driver": "GTiff",
                           "compress": compress,
                           "nodata": nodata,
                           "height": self._height,
                           "width": self._width,
                           "dtype": result_layer_i.dtype,
                           "transform": src_layer.window_transform(self._window)})
            with rasterio.open(dst_path_chunk, "w", **kwargs) as dst:
                dst.write(result_layer_i[:, :, 0], 1)
1200943
def toArray(self):
        
        A = np.zeros((self.numRows, self.numCols), dtype=np.float64, order='F')
        for k in xrange(self.colPtrs.size - 1):
            startptr = self.colPtrs[k]
            endptr = self.colPtrs[k + 1]
            if self.isTransposed:
                A[k, self.rowIndices[startptr:endptr]] = self.values[startptr:endptr]
            else:
                A[self.rowIndices[startptr:endptr], k] = self.values[startptr:endptr]
        return A
1056267
def unbroadcast_numpy_to(array, shape):
  
  axis = create_unbroadcast_axis(shape, numpy.shape(array))
  return numpy.reshape(numpy.sum(array, axis=axis), shape)
1325456
public static INDArray toVector(Vector arr) {
        return Nd4j.create(Nd4j.createBuffer(arr.toArray()));
    }
951775
def __init_from_list_np2d(self, mats, params_str, ref_dataset):
        """"""
        ncol = mats[0].shape[1]
        nrow = np.zeros((len(mats),), np.int32)
        if mats[0].dtype == np.float64:
            ptr_data = (ctypes.POINTER(ctypes.c_double) * len(mats))()
        else:
            ptr_data = (ctypes.POINTER(ctypes.c_float) * len(mats))()

        holders = []
        type_ptr_data = None

        for i, mat in enumerate(mats):
            if len(mat.shape) != 2:
                raise ValueError('Input numpy.ndarray must be 2 dimensional')

            if mat.shape[1] != ncol:
                raise ValueError('Input arrays must have same number of columns')

            nrow[i] = mat.shape[0]

            if mat.dtype == np.float32 or mat.dtype == np.float64:
                mats[i] = np.array(mat.reshape(mat.size), dtype=mat.dtype, copy=False)
            else:
                # change non-float data to float data, need to copy
                mats[i] = np.array(mat.reshape(mat.size), dtype=np.float32)

            chunk_ptr_data, chunk_type_ptr_data, holder = c_float_array(mats[i])
            if type_ptr_data is not None and chunk_type_ptr_data != type_ptr_data:
                raise ValueError('Input chunks must have same type')
            ptr_data[i] = chunk_ptr_data
            type_ptr_data = chunk_type_ptr_data
            holders.append(holder)

        self.handle = ctypes.c_void_p()
        _safe_call(_LIB.LGBM_DatasetCreateFromMats(
            ctypes.c_int(len(mats)),
            ctypes.cast(ptr_data, ctypes.POINTER(ctypes.POINTER(ctypes.c_double))),
            ctypes.c_int(type_ptr_data),
            nrow.ctypes.data_as(ctypes.POINTER(ctypes.c_int32)),
            ctypes.c_int(ncol),
            ctypes.c_int(C_API_IS_ROW_MAJOR),
            c_str(params_str),
            ref_dataset,
            ctypes.byref(self.handle)))
        return self
136266
def toArray(self):
        
        if self.isTransposed:
            return np.asfortranarray(
                self.values.reshape((self.numRows, self.numCols)))
        else:
            return self.values.reshape((self.numRows, self.numCols), order='F')
99745
def to_java_doubles(m):
    '''
    
    '''
    global _java
    if _java is None: _init_registration()
    m = np.asarray(m)
    dims = len(m.shape)
    if dims > 2: raise ValueError('1D and 2D arrays supported only')
    bindat = serialize_numpy(m, 'd')
    return (_java.jvm.nben.util.Numpy.double2FromBytes(bindat) if dims == 2
            else _java.jvm.nben.util.Numpy.double1FromBytes(bindat))
1050554
def _coerce_to_ndarray(self):
        

        # TODO(jreback) make this better
        data = self._data.astype(object)
        data[self._mask] = self._na_value
        return data
1292559
def JSON_NumpyArrayEncoder(obj):
    ''''''
    if isinstance(obj, np.ndarray):
        return {'numpyArray': obj.tolist(),
                'dtype': obj.dtype.__str__()}
    elif isinstance(obj, np.generic):
        return np.asscalar(obj)
    else:
        print type(obj)
        raise TypeError(repr(obj) + " is not JSON serializable")
578881
def ndarray_to_imagedatadict(nparr):
    
    ret = {}
    dm_type = None
    for k, v in iter(dm_image_dtypes.items()):
        if v[1] == nparr.dtype.type:
            dm_type = k
            break
    if dm_type is None and nparr.dtype == numpy.uint8 and nparr.shape[-1] in (3, 4):
        ret["DataType"] = 23
        ret["PixelDepth"] = 4
        if nparr.shape[2] == 4:
            rgb_view = nparr.view(numpy.int32).reshape(nparr.shape[:-1])  # squash the color into uint32
        else:
            assert nparr.shape[2] == 3
            rgba_image = numpy.empty(nparr.shape[:-1] + (4,), numpy.uint8)
            rgba_image[:,:,0:3] = nparr
            rgba_image[:,:,3] = 255
            rgb_view = rgba_image.view(numpy.int32).reshape(rgba_image.shape[:-1])  # squash the color into uint32
        ret["Dimensions"] = list(rgb_view.shape[::-1])
        ret["Data"] = parse_dm3.array.array(platform_independent_char(rgb_view.dtype), rgb_view.flatten())
    else:
        ret["DataType"] = dm_type
        ret["PixelDepth"] = nparr.dtype.itemsize
        ret["Dimensions"] = list(nparr.shape[::-1])
        if nparr.dtype.type in np_to_structarray_map:
            types = np_to_structarray_map[nparr.dtype.type]
            ret["Data"] = parse_dm3.structarray(types)
            ret["Data"].raw_data = bytes(numpy.array(nparr, copy=False).data)
        else:
            ret["Data"] = parse_dm3.array.array(platform_independent_char(nparr.dtype), numpy.array(nparr, copy=False).flatten())
    return ret
1343145
def _row_sparse_ndarray_from_definition(data, indices, shape=None, ctx=None,
                                        dtype=None, indices_type=None):
    """"""
    storage_type = 'row_sparse'
    # context
    ctx = current_context() if ctx is None else ctx
    # types
    dtype = _prepare_default_dtype(data, dtype)
    indices_type = _STORAGE_AUX_TYPES[storage_type][0] if indices_type is None else indices_type
    # prepare src array and types
    data = _prepare_src_array(data, dtype)
    indices = _prepare_src_array(indices, indices_type)

    # TODO(junwu): Convert data, indptr, and indices to mxnet NDArrays
    # if they are not for now. In the future, we should provide a c-api
    # to accept np.ndarray types to copy from to result.data and aux_data
    if not isinstance(data, NDArray):
        data = _array(data, ctx, dtype)
    if not isinstance(indices, NDArray):
        indices = _array(indices, ctx, indices_type)
    if shape is None:
        num_indices = indices.shape[0]
        if num_indices == 0:
            raise ValueError('invalid shape')
        dim0 = indices[num_indices - 1].asscalar() + 1
        shape = (dim0, ) + data.shape[1:]
    # verify shapes
    if data.ndim != len(shape) or indices.ndim != 1 or np.prod(shape[1:]) == 0:
        raise ValueError("invalid shape")
    result = RowSparseNDArray(_new_alloc_handle(storage_type, shape, ctx, False, dtype,
                                                [indices_type], [indices.shape]))
    check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, data.handle, ctypes.c_int(-1)))
    check_call(_LIB.MXNDArraySyncCopyFromNDArray(result.handle, indices.handle, ctypes.c_int(0)))
    return result
170401
def asarray2d(a):
    """"""
    arr = np.asarray(a)
    if arr.ndim == 1:
        arr = arr.reshape(-1, 1)
    return arr
617155
def _load_into_numpy(sf, np_array, start, end, strides=None, shape=None):
    """"""
    np_array[:] = 0.0
    np_array_2d = np_array.reshape((np_array.shape[0], np_array.shape[1] * np_array.shape[2]))
    _extensions.sframe_load_to_numpy(sf, np_array.ctypes.data,
                                     np_array_2d.strides, np_array_2d.shape,
                                     start, end)
309571
def transform_array_to_list(array):
    ''' 

    '''
    if (array.dtype.kind in ('u', 'i', 'f') and (~np.isfinite(array)).any()):
        transformed = array.astype('object')
        transformed[np.isnan(array)] = 'NaN'
        transformed[np.isposinf(array)] = 'Infinity'
        transformed[np.isneginf(array)] = '-Infinity'
        return transformed.tolist()
    elif (array.dtype.kind == 'O' and pd and pd.isnull(array).any()):
        transformed = array.astype('object')
        transformed[pd.isnull(array)] = 'NaN'
        return transformed.tolist()
    return array.tolist()
187568
def to_0d_array(value: Any) -> np.ndarray:
    
    if np.isscalar(value) or (isinstance(value, np.ndarray) and
                              value.ndim == 0):
        return np.array(value)
    else:
        return to_0d_object_array(value)
856130
def get_nd4j_dtype(np_dtype):
    
    if type(np_dtype) == type:
        np_dtype = np_dtype.__name__
    elif type(np_dtype) == np.dtype:
        np_dtype = np_dtype.name
    mapping = {
        'float64': 'double',
        'float32': 'float',
        'float16': 'half'
    }
    nd4j_dtype = mapping.get(np_dtype)
    if not nd4j_dtype:
        raise Exception('Invalid numpy data type : ' + np_dtype)
    return nd4j_dtype