Creates a new ndarray of the specified shape and datatype, storing
	it in memory if there is sufficient available space or else using
	a memory-mapped temporary file to provide the underlying buffer.
429137
def arrayFactory(shape, dtype=float):
	
	
	# Determine the number of bytes required to store the array
	requiredBytes = _requiredSize(shape, dtype)
	
	# Determine if there is sufficient available memory
	vmem = psutil.virtual_memory()
	if vmem.available > requiredBytes:
		return np.ndarray(shape=shape, dtype=dtype)
	else:
		return TempfileBackedArray(shape=shape, dtype=dtype)
1049778
public static Pair<INDArray, ByteBuffer> toArrayAndByteBuffer(ByteBuffer buffer, int offset) {
        ByteBuffer byteBuffer = buffer.hasArray() ? ByteBuffer.allocateDirect(buffer.array().length).put(buffer.array())
                .order(ByteOrder.nativeOrder()) : buffer.order(ByteOrder.nativeOrder());
        //bump the byte buffer to the proper position
        byteBuffer.position(offset);
        int rank = byteBuffer.getInt();
        if (rank < 0)
            throw new IllegalStateException("Found negative integer. Corrupt serialization?");
        //get the shape buffer length to create the shape information buffer
        int shapeBufferLength = Shape.shapeInfoLength(rank);
        //create the ndarray shape information
        DataBuffer shapeBuff = Nd4j.createBufferDetached(new int[shapeBufferLength]);

        //compute the databuffer opType from the index
        DataType type = DataType.values()[byteBuffer.getInt()];
        for (int i = 0; i < shapeBufferLength; i++) {
            shapeBuff.put(i, byteBuffer.getLong());
        }

        //after the rank,data opType, shape buffer (of length shape buffer length) * sizeof(int)
        if (type != DataType.COMPRESSED) {
            ByteBuffer slice = byteBuffer.slice();
            //wrap the data buffer for the last bit
            // FIXME: int cast
            DataBuffer buff = Nd4j.createBuffer(slice, type, (int) Shape.length(shapeBuff));
            //advance past the data
            int position = byteBuffer.position() + (buff.getElementSize() * (int) buff.length());
            byteBuffer.position(position);
            //create the final array
            //TODO: see how to avoid dup here
            INDArray arr = Nd4j.createArrayFromShapeBuffer(buff.dup(), shapeBuff.dup());
            return Pair.of(arr, byteBuffer);
        } else {
            CompressionDescriptor compressionDescriptor = CompressionDescriptor.fromByteBuffer(byteBuffer);
            ByteBuffer slice = byteBuffer.slice();
            //ensure that we only deal with the slice of the buffer that is actually the data
            BytePointer byteBufferPointer = new BytePointer(slice);
            //create a compressed array based on the rest of the data left in the buffer
            CompressedDataBuffer compressedDataBuffer =
                    new CompressedDataBuffer(byteBufferPointer, compressionDescriptor);
            //TODO: see how to avoid dup()
            INDArray arr = Nd4j.createArrayFromShapeBuffer(compressedDataBuffer.dup(), shapeBuff.dup());
            //advance past the data
            int compressLength = (int) compressionDescriptor.getCompressedLength();
            byteBuffer.position(byteBuffer.position() + compressLength);
            return Pair.of(arr, byteBuffer);
        }

    }
505503
def _new_alloc_handle(stype, shape, ctx, delay_alloc, dtype, aux_types, aux_shapes=None):
    
    hdl = NDArrayHandle()
    for aux_t in aux_types:
        if np.dtype(aux_t) != np.dtype("int64"):
            raise NotImplementedError("only int64 is supported for aux types")
    aux_type_ids = [int(_DTYPE_NP_TO_MX[np.dtype(aux_t).type]) for aux_t in aux_types]
    aux_shapes = [(0,) for aux_t in aux_types] if aux_shapes is None else aux_shapes
    aux_shape_lens = [len(aux_shape) for aux_shape in aux_shapes]
    aux_shapes = py_sum(aux_shapes, ())
    num_aux = mx_uint(len(aux_types))
    check_call(_LIB.MXNDArrayCreateSparseEx(
        ctypes.c_int(int(_STORAGE_TYPE_STR_TO_ID[stype])),
        c_array_buf(mx_uint, native_array('I', shape)),
        mx_uint(len(shape)),
        ctypes.c_int(ctx.device_typeid),
        ctypes.c_int(ctx.device_id),
        ctypes.c_int(int(delay_alloc)),
        ctypes.c_int(int(_DTYPE_NP_TO_MX[np.dtype(dtype).type])),
        num_aux,
        c_array_buf(ctypes.c_int, native_array('i', aux_type_ids)),
        c_array_buf(mx_uint, native_array('I', aux_shape_lens)),
        c_array_buf(mx_uint, native_array('I', aux_shapes)),
        ctypes.byref(hdl)))
    return hdl
392031
def create(input_width, input_height, input_channels=1, output_dim=512):
    
    def instantiate(**_):
        return NatureCnn(
            input_width=input_width, input_height=input_height, input_channels=input_channels,
            output_dim=output_dim
        )

    return ModelFactory.generic(instantiate)
899611
def _frombuffer(ptr, frames, channels, dtype):
    """"""
    framesize = channels * dtype.itemsize
    data = np.frombuffer(ffi.buffer(ptr, frames * framesize), dtype=dtype)
    data.shape = -1, channels
    return data
289017
public static INDArray buildFromData(List<DataPoint> data) {
        INDArray ret = Nd4j.create(data.size(), data.get(0).getD());
        for (int i = 0; i < ret.slices(); i++)
            ret.putSlice(i, data.get(i).getPoint());
        return ret;
    }
1280773
public SDVariable fill(SDVariable shape, org.nd4j.linalg.api.buffer.DataType dataType, double value) {
        return fill(null, shape, dataType, value);
    }
491890
def _from_numpy(np_array):
    

    # Convert the numpy array to nd4j context dtype
    required_dtype = get_np_dtype(get_context_dtype())
    if np_array.dtype != required_dtype:
        raise Exception("{} is required, got {}".format(
            repr(required_dtype), repr(np_array.dtype)))

    # Nd4j does not have 1-d vectors.
    # So we add a dummy dimension.
    if np_array.ndim == 1:
        np_array = np.expand_dims(np_array, 0)

    # We have to maintain references to all incoming
    # numpy arrays. Else they will get GCed

    # creates a Nd4j array from a numpy array
    # To create an Nd4j array, we need 3 things:
    # buffer, strides, and shape

    # Get the buffer
    # A buffer is basically an array. To get the buffer object
    # we need a pointer to the first element and the size.
    pointer_address, _ = np_array.__array_interface__['data']
    _refs.append(np_array)
    pointer = native_ops.pointerForAddress(pointer_address)
    size = np_array.size
    mapping = {
        np.float64: DoublePointer,
        np.float32: FloatPointer,
    }
    pointer = mapping[required_dtype](pointer)
    buff = Nd4j.createBuffer(pointer, size)
    assert buff.address() == pointer_address
    _refs.append(buff)
    # Get the strides
    # strides = tuple of bytes to step in each
    # dimension when traversing an array.
    elem_size = buff.getElementSize()
    # Make sure word size is same in both python
    # and java worlds
    assert elem_size == np_array.dtype.itemsize
    strides = np_array.strides
    # numpy uses byte wise strides. We have to
    # convert it to word wise strides.
    strides = [dim / elem_size for dim in strides]

    # Finally, shape:
    shape = np_array.shape

    nd4j_array = Nd4j.create(buff, shape, strides, 0)
    assert buff.address() == nd4j_array.data().address()
    return nd4j_array
9178
def valarray(shape, value=np.NaN, typecode=None):
    """"""
    if typecode is None:
        typecode = bool
    out = np.ones(shape, dtype=typecode) * value

    if not isinstance(out, np.ndarray):
        out = np.asarray(out)
    return out
1204643
def _prepare_default_dtype(src_array, dtype):
    """"""
    if dtype is None:
        if isinstance(src_array, (NDArray, np.ndarray)):
            dtype = src_array.dtype
        elif spsp and isinstance(src_array, spsp.csr.csr_matrix):
            dtype = src_array.dtype
        else:
            dtype = mx_real_t
    return dtype
1051015
def _simple_new(cls, values, name=None, dtype=None, **kwargs):
        
        if not hasattr(values, 'dtype'):
            if (values is None or not len(values)) and dtype is not None:
                values = np.empty(0, dtype=dtype)
            else:
                values = np.array(values, copy=False)
                if is_object_dtype(values):
                    values = cls(values, name=name, dtype=dtype,
                                 **kwargs)._ndarray_values

        if isinstance(values, (ABCSeries, ABCIndexClass)):
            # Index._data must always be an ndarray.
            # This is no-copy for when _values is an ndarray,
            # which should be always at this point.
            values = np.asarray(values._values)

        result = object.__new__(cls)
        result._data = values
        # _index_data is a (temporary?) fix to ensure that the direct data
        # manipulation we do in `_libs/reduction.pyx` continues to work.
        # We need access to the actual ndarray, since we're messing with
        # data buffers and strides. We don't re-use `_ndarray_values`, since
        # we actually set this value too.
        result._index_data = values
        result.name = name
        for k, v in kwargs.items():
            setattr(result, k, v)
        return result._reset_identity()
65584
def save(self, obj):
        
        if isinstance(obj, self.np.ndarray) and not obj.dtype.hasobject:
            # Compute a hash of the object
            # The update function of the hash requires a c_contiguous buffer.
            if obj.shape == ():
                # 0d arrays need to be flattened because viewing them as bytes
                # raises a ValueError exception.
                obj_c_contiguous = obj.flatten()
            elif obj.flags.c_contiguous:
                obj_c_contiguous = obj
            elif obj.flags.f_contiguous:
                obj_c_contiguous = obj.T
            else:
                # Cater for non-single-segment arrays: this creates a
                # copy, and thus aleviates this issue.
                # XXX: There might be a more efficient way of doing this
                obj_c_contiguous = obj.flatten()

            # memoryview is not supported for some dtypes, e.g. datetime64, see
            # https://github.com/numpy/numpy/issues/4983. The
            # workaround is to view the array as bytes before
            # taking the memoryview.
            self._hash.update(
                self._getbuffer(obj_c_contiguous.view(self.np.uint8)))

            # We store the class, to be able to distinguish between
            # Objects with the same binary content, but different
            # classes.
            if self.coerce_mmap and isinstance(obj, self.np.memmap):
                # We don't make the difference between memmap and
                # normal ndarrays, to be able to reload previously
                # computed results with memmap.
                klass = self.np.ndarray
            else:
                klass = obj.__class__
            # We also return the dtype and the shape, to distinguish
            # different views on the same data with different dtypes.

            # The object will be pickled by the pickler hashed at the end.
            obj = (klass, ('HASHED', obj.dtype, obj.shape, obj.strides))
        elif isinstance(obj, self.np.dtype):
            # Atomic dtype objects are interned by their default constructor:
            # np.dtype('f8') is np.dtype('f8')
            # This interning is not maintained by a
            # pickle.loads + pickle.dumps cycle, because __reduce__
            # uses copy=True in the dtype constructor. This
            # non-deterministic behavior causes the internal memoizer
            # of the hasher to generate different hash values
            # depending on the history of the dtype object.
            # To prevent the hash from being sensitive to this, we use
            # .descr which is a full (and never interned) description of
            # the array dtype according to the numpy doc.
            klass = obj.__class__
            obj = (klass, ('HASHED', obj.descr))
        Hasher.save(self, obj)
1181687
def newarray(self, length, value=0):
        """"""
        if self.bitdepth > 8:
            return array('H', [value] * length)
        else:
            return bytearray([value] * length)
163024
def make_hdu(self, data, **kwargs):
        
        shape = data.shape
        extname = kwargs.get('extname', self.conv.extname)

        if shape[-1] != self._npix:
            raise Exception(
                "Size of data array does not match number of pixels")
        cols = []

        if self._ipix is not None:
            cols.append(fits.Column(self.conv.idxstring, "J", array=self._ipix))

        if self.conv.convname == 'FGST_SRCMAP_SPARSE':
            nonzero = data.nonzero()
            nfilled = len(nonzero[0])
            if len(shape) == 1:
                cols.append(fits.Column("PIX", "J", array=nonzero[0].astype(int)))
                cols.append(fits.Column("VALUE", "E", array=data.flat[nonzero].astype(float).reshape(nfilled)))
            elif len(shape) == 2:
                keys = self._npix * nonzero[0] + nonzero[1]
                cols.append(fits.Column("PIX", "J", array=nonzero[1].reshape(nfilled)))
                cols.append(fits.Column("CHANNEL", "I", array=nonzero[0].reshape(nfilled)))
                cols.append(fits.Column("VALUE", "E",
                                        array=data.flat[keys].astype(float).reshape(nfilled)))
            else:
                raise Exception("HPX.write_fits only handles 1D and 2D maps")

        else:
            if len(shape) == 1:
                cols.append(fits.Column(self.conv.colname(
                    indx=self.conv.firstcol), "E", array=data.astype(float)))
            elif len(shape) == 2:
                for i in range(shape[0]):
                    cols.append(fits.Column(self.conv.colname(
                        indx=i + self.conv.firstcol), "E", array=data[i].astype(float)))
            else:
                raise Exception("HPX.write_fits only handles 1D and 2D maps")

        header = self.make_header()
        hdu = fits.BinTableHDU.from_columns(cols, header=header, name=extname)

        return hdu
578803
def create_image(data: np.ndarray, colormap, data_min=None, data_max=None, normalize=True) -> QImage:
        
        image_data = Spectrogram.apply_bgra_lookup(data, colormap, data_min, data_max, normalize)

        if not image_data.flags['C_CONTIGUOUS']:
            logger.debug("Array was not C_CONTIGUOUS. Converting it.")
            image_data = np.ascontiguousarray(image_data)

        try:
            # QImage constructor needs inverted row/column order
            image = QImage(image_data.ctypes.data, image_data.shape[1], image_data.shape[0], QImage.Format_ARGB32)
        except Exception as e:
            logger.error("could not create image " + str(e))
            return QImage()

        image.data = image_data
        return image
1298550
public static NDArrayMessageChunk fromBuffer(ByteBuffer byteBuffer, NDArrayMessage.MessageType type) {
        int numChunks = byteBuffer.getInt();
        int chunkSize = byteBuffer.getInt();
        int idLength = byteBuffer.getInt();
        byte[] id = new byte[idLength];
        byteBuffer.get(id);
        String idString = new String(id);
        int index = byteBuffer.getInt();
        ByteBuffer firstData = byteBuffer.slice();
        NDArrayMessageChunk chunk = NDArrayMessageChunk.builder().chunkSize(chunkSize).numChunks(numChunks)
                        .data(firstData).messageType(type).id(idString).chunkIndex(index).build();
        return chunk;

    }
429843
def _copy_with_dtype(data, dtype):
    
    result = np.empty(data.shape, dtype)
    result[...] = data
    return result