Extend an extensible dataset with an array of a compatible dtype.

    :param dset: an h5py dataset
    :param array: an array of length L
    :returns: the total length of the dataset (i.e. initial length + L)
337834
def extend(dset, array, **attrs):
    
    length = len(dset)
    if len(array) == 0:
        return length
    newlength = length + len(array)
    if array.dtype.name == 'object':  # vlen array
        shape = (newlength,) + preshape(array[0])
    else:
        shape = (newlength,) + array.shape[1:]
    dset.resize(shape)
    dset[length:newlength] = array
    for key, val in attrs.items():
        dset.attrs[key] = val
    return newlength
1161841
def array_bytes(array):
    
    return np.product(array.shape)*np.dtype(array.dtype).itemsize
1338147
def read_array(fd, n_row, n_col, dtype):
    
    if n_col is None:
        idx = fd.tell()
        row = fd.readline().split()
        fd.seek(idx)
        n_col = len(row)

    count = n_row * n_col
    val = nm.fromfile(fd, sep=' ', count=count)

    if val.shape[0] < count:
        raise ValueError('(%d, %d) array reading failed!' % (n_row, n_col))

    val = nm.asarray(val, dtype=dtype)
    val.shape = (n_row, n_col)

    return val
142126
def expand(self, key_array):
        

        if len(key_array) != self._n:
            raise RuntimeError('expand(): key size ' + str(len(key_array)) + ' is invalid')

        # First n bytes are copied from key. Copy prevents inplace modification of original key
        new_key = list(key_array)

        rcon_iteration = 1
        len_new_key = len(new_key)

        # There are several parts of the code below that could be done with tidy list comprehensions like
        # the one I put in _core, but I left this alone for readability.

        # Grow the key until it is the correct length
        while len_new_key < self._b:

            # Copy last 4 bytes of extended key, apply _core function order i, increment i(rcon_iteration),
            # xor with 4 bytes n bytes from end of extended key
            t = new_key[-4:]
            t = self._core(t, rcon_iteration)
            rcon_iteration += 1
            t = self._xor_list(t, new_key[-self._n : -self._n + 4])# self._n_bytes_before(len_new_key, new_key))
            new_key.extend(t)
            len_new_key += 4

            # Run three passes of 4 byte expansion using copy of 4 byte tail of extended key
            # which is then xor'd with 4 bytes n bytes from end of extended key
            for j in range(3):
                t = new_key[-4:]
                t = self._xor_list(t, new_key[-self._n : -self._n + 4])
                new_key.extend(t)
                len_new_key += 4

            # If key length is 256 and key is not complete, add 4 bytes tail of extended key
            # run through sbox before xor with 4 bytes n bytes from end of extended key
            if self._key_length == 256 and len_new_key < self._b:
                t = new_key[-4:]
                t2=[]
                for x in t:
                    t2.append(aes_tables.sbox[x])
                t = self._xor_list(t2, new_key[-self._n : -self._n + 4])
                new_key.extend(t)
                len_new_key += 4

            # If key length is 192 or 256 and key is not complete, run 2 or 3 passes respectively
            # of 4 byte tail of extended key xor with 4 bytes n bytes from end of extended key
            if self._key_length != 128 and len_new_key < self._b:
                if self._key_length == 192:
                    r = range(2)
                else:
                    r = range(3)

                for j in r:
                    t = new_key[-4:]
                    t = self._xor_list(t, new_key[-self._n : -self._n + 4])
                    new_key.extend(t)
                    len_new_key += 4

        return new_key
498936
def _extend_data(self, datapoint, new_data):
        
        if new_data:
            try:
                self.data[datapoint].extend(new_data)
            except KeyError:
                self.data[datapoint] = new_data
1286915
def addDataset(self, dataset):
        
        id_ = dataset.getId()
        self._datasetIdMap[id_] = dataset
        self._datasetNameMap[dataset.getLocalId()] = dataset
        self._datasetIds.append(id_)
312401
def rec_append_fields(rec, names, arrs, dtypes=None):
    
    if (not isstring(names) and iterable(names) and len(names) and isstring(names[0])):
        if len(names) != len(arrs):
            raise ValueError("number of arrays do not match number of names")
    else:  # we have only 1 name and 1 array
        names = [names]
        arrs = [arrs]
    arrs = list(map(np.asarray, arrs))
    if dtypes is None:
        dtypes = [a.dtype for a in arrs]
    elif not iterable(dtypes):
        dtypes = [dtypes]
    if len(arrs) != len(dtypes):
        if len(dtypes) == 1:
            dtypes = dtypes * len(arrs)
        else:
            raise ValueError("dtypes must be None, a single dtype or a list")
    old_dtypes = rec.dtype.descr
    if six.PY2:
        old_dtypes = [(name.encode('utf-8'), dt) for name, dt in old_dtypes]
    newdtype = np.dtype(old_dtypes + list(zip(names, dtypes)))
    newrec = np.recarray(rec.shape, dtype=newdtype)
    for field in rec.dtype.fields:
        newrec[field] = rec[field]
    for name, arr in zip(names, arrs):
        newrec[name] = arr
    return newrec
629712
def broadcast_1d_array(arr, ndim, axis=1):
    
    ext_arr = arr
    for i in range(ndim - 1):
        ext_arr = np.expand_dims(ext_arr, axis=axis)
    return ext_arr
1182156
def cast(cls, datasets, datatype=None, cast_type=None):
        
        datatype = datatype or cls.datatype
        cast = []
        for ds in datasets:
            if cast_type is not None or ds.interface.datatype != datatype:
                ds = ds.clone(ds, datatype=[datatype], new_type=cast_type)
            cast.append(ds)
        return cast
807210
def _make_dataset(cls, coords):
        
        class Slice(cls._SliceType):
            extra_coords = coords

        Slice.__name__ = '%s.slice(%s)' % (
            cls.__name__,
            ', '.join('%s=%r' % item for item in coords.items()),
        )
        return Slice
1173762
def get_extra_ds(dataset):
    """"""
    ds_collection = {}
    for ds in dataset.attrs.get('ancillary_variables', []):
        ds_collection.update(get_extra_ds(ds))
    ds_collection[dataset.attrs['name']] = dataset

    return ds_collection
373181
def write_array_metadata(dataset, array):
    
    for attr in ('unit',) + array._metadata_slots:
        # format attribute
        try:
            value = _format_metadata_attribute(
                getattr(array, '_%s' % attr, None))
        except IgnoredAttribute:
            continue

        # store attribute
        try:
            dataset.attrs[attr] = value
        except (TypeError, ValueError, RuntimeError) as exc:
            exc.args = ("Failed to store {} ({}) for {}: {}".format(
                attr, type(value).__name__, type(array).__name__, str(exc)))
            raise
468225
def get_dataset(self, dataset_id, ds_info):
        """"""
        msg = self._get_message(ds_info)
        ds_info = self.get_metadata(msg, ds_info)
        fill = msg['missingValue']
        data = msg.values.astype(np.float32)
        if msg.valid_key('jScansPositively') and msg['jScansPositively'] == 1:
            data = data[::-1]

        if isinstance(data, np.ma.MaskedArray):
            data = data.filled(np.nan)
            data = da.from_array(data, chunks=CHUNK_SIZE)
        else:
            data[data == fill] = np.nan
            data = da.from_array(data, chunks=CHUNK_SIZE)

        return xr.DataArray(data, attrs=ds_info, dims=('y', 'x'))
731388
def _load_dataset_with_area(self, dsid, coords):
        """"""
        file_handlers = self._get_file_handlers(dsid)
        if not file_handlers:
            return

        area = self._load_dataset_area(dsid, file_handlers, coords)

        try:
            ds = self._load_dataset_data(file_handlers, dsid)
        except (KeyError, ValueError) as err:
            logger.exception("Could not load dataset '%s': %s", dsid, str(err))
            return None

        if area is not None:
            ds.attrs['area'] = area
            calc_coords = (('x' not in ds.coords) or('y' not in ds.coords)) and hasattr(area, 'get_proj_vectors_dask')
            if calc_coords and hasattr(area, 'get_proj_vectors'):
                ds['x'], ds['y'] = area.get_proj_vectors()
            elif calc_coords:
                # older pyresample with dask-only method
                ds['x'], ds['y'] = area.get_proj_vectors_dask(CHUNK_SIZE)
        return ds
34671
def concatenate(cls, datasets, datatype=None, new_type=None):
        
        from . import Dataset, default_datatype
        new_type = new_type or Dataset
        if isinstance(datasets, NdMapping):
            dimensions = datasets.kdims
            keys, datasets = zip(*datasets.data.items())
        elif isinstance(datasets, list) and all(not isinstance(v, tuple) for v in datasets):
            # Allow concatenating list of datasets (by declaring no dimensions and keys)
            dimensions, keys = [], [()]*len(datasets)
        else:
            raise DataError('Concatenation only supported for NdMappings '
                            'and lists of Datasets, found %s.' % type(datasets).__name__)

        template = datasets[0]
        datatype = datatype or template.interface.datatype

        # Handle non-general datatypes by casting to general type
        if datatype == 'array':
            datatype = default_datatype
        elif datatype == 'image':
            datatype = 'grid'

        if len(datasets) > 1 and not dimensions and cls.interfaces[datatype].gridded:
            raise DataError('Datasets with %s datatype cannot be concatenated '
                            'without defining the dimensions to concatenate along. '
                            'Ensure you pass in a NdMapping (e.g. a HoloMap) '
                            'of Dataset types, not a list.' % datatype)

        datasets = template.interface.cast(datasets, datatype)
        template = datasets[0]
        data = list(zip(keys, datasets)) if keys else datasets
        concat_data = template.interface.concat(data, dimensions, vdims=template.vdims)
        return template.clone(concat_data, kdims=dimensions+template.kdims, new_type=new_type)
971209
def get_dataset(self, dataset_id, ds_info, xslice=slice(None), yslice=slice(None)):
        """"""
        var_path = ds_info.get('file_key', '{}'.format(dataset_id.name))
        metadata = self.get_metadata(dataset_id, ds_info)
        shape = metadata['shape']
        file_shape = self[var_path + '/shape']
        if isinstance(shape, tuple) and len(shape) == 2:
            # 2D array
            if xslice.start is not None:
                shape = (shape[0], xslice.stop - xslice.start)
            if yslice.start is not None:
                shape = (yslice.stop - yslice.start, shape[1])
        elif isinstance(shape, tuple) and len(shape) == 1 and yslice.start is not None:
            shape = ((yslice.stop - yslice.start) / yslice.step,)
        metadata['shape'] = shape

        valid_min = self[var_path + '/attr/valid_min']
        valid_max = self[var_path + '/attr/valid_max']
        # no need to check fill value since we are using valid min/max
        scale_factor = self.get(var_path + '/attr/scale_factor')
        add_offset = self.get(var_path + '/attr/add_offset')

        if isinstance(file_shape, tuple) and len(file_shape) == 3:
            data = self[var_path][0, yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 2:
            data = self[var_path][yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 1:
            data = self[var_path][yslice]
        else:
            data = self[var_path]
        data = data.where((data >= valid_min) & (data <= valid_max))
        if scale_factor is not None:
            data = data * scale_factor + add_offset

        if ds_info.get('cloud_clear', False):
            # clear-sky if bit 15-16 are 00
            clear_sky_mask = (self['l2p_flags'][0] & 0b1100000000000000) != 0
            data = data.where(~clear_sky_mask)

        data.attrs.update(metadata)
        return data
278120
def average_arrays(arrays: List[mx.nd.NDArray]) -> mx.nd.NDArray:
    
    if not arrays:
        raise ValueError("arrays is empty.")
    if len(arrays) == 1:
        return arrays[0]
    check_condition(all(arrays[0].shape == a.shape for a in arrays), "nd array shapes do not match")
    return mx.nd.add_n(*arrays) / len(arrays)