Convert a numpy NPZ file to h5features.
438624
def npz_convert(self, infile, item):
        """"""
        data = np.load(infile)
        labels = self._labels(data)
        features = data['features']
        self._write(item, labels, features)
54259
def dict_to_hdf5(dict_like, h5group):
    
    # Write attributes
    for key, value in dict_like.items():
        if value is not None:
            h5group.attrs[str(key)] = value
1368251
def setupimgh5(f: Union[Path, h5py.File],
               Nframetotal: int, Nrow: int, Ncol: int, dtype=np.uint16,
               writemode='r+', key='/rawimg', cmdlog: str=None):
    
    if isinstance(f, (str, Path)):  # assume new HDF5 file wanted
        f = Path(f).expanduser()
        if f.is_dir():
            raise IOError(
                'must provide specific HDF5 filename, not just a directory')

        print('creating', f)
        with h5py.File(f, writemode) as F:
            setupimgh5(F, Nframetotal, Nrow, Ncol, dtype, writemode, key)

    elif isinstance(f, h5py.File):
        h = f.create_dataset(key,
                             shape=(Nframetotal, Nrow, Ncol),
                             dtype=dtype,
                             chunks=(1, Nrow, Ncol),  # each image is a chunk
                             compression='gzip',
                             # no difference in filesize from 1 to 5, except much faster to use lower numbers!
                             compression_opts=1,
                             shuffle=True,
                             fletcher32=True,
                             track_times=True)
        h.attrs["CLASS"] = np.string_("IMAGE")
        h.attrs["IMAGE_VERSION"] = np.string_("1.2")
        h.attrs["IMAGE_SUBCLASS"] = np.string_("IMAGE_GRAYSCALE")
        h.attrs["DISPLAY_ORIGIN"] = np.string_("LL")
        h.attrs['IMAGE_WHITE_IS_ZERO'] = np.uint8(0)

        if cmdlog and isinstance(cmdlog, str):
            f['/cmdlog'] = cmdlog
    else:
        raise TypeError(
            f'{type(f)} is not correct, must be filename or h5py.File HDF5 file handle')
1314866
def write_to_files_np(features, tokenizer, max_seq_length,
                      max_predictions_per_seq, output_files):
    # pylint: disable=unused-argument
    """"""
    next_sentence_labels = []
    valid_lengths = []

    assert len(output_files) == 1, 'numpy format only support single output file'
    output_file = output_files[0]
    (input_ids, segment_ids, masked_lm_positions, masked_lm_ids,
     masked_lm_weights, next_sentence_labels, valid_lengths) = features
    total_written = len(next_sentence_labels)

    # store variable length numpy array object directly.
    outputs = collections.OrderedDict()
    outputs['input_ids'] = np.array(input_ids, dtype=object)
    outputs['segment_ids'] = np.array(segment_ids, dtype=object)
    outputs['masked_lm_positions'] = np.array(masked_lm_positions, dtype=object)
    outputs['masked_lm_ids'] = np.array(masked_lm_ids, dtype=object)
    outputs['masked_lm_weights'] = np.array(masked_lm_weights, dtype=object)
    outputs['next_sentence_labels'] = np.array(next_sentence_labels, dtype='int32')
    outputs['valid_lengths'] = np.array(valid_lengths, dtype='int32')

    np.savez_compressed(output_file, **outputs)
    logging.info('Wrote %d total instances', total_written)
514440
def cfloat32_array_to_numpy(cptr, length):
    """"""
    if isinstance(cptr, ctypes.POINTER(ctypes.c_float)):
        return np.fromiter(cptr, dtype=np.float32, count=length)
    else:
        raise RuntimeError('Expected float pointer')
1280531
def segments_from_numpy(segments):
    """"""
    segments = segments if SEGMENTS_DIRECTION == 0 else segments.tranpose()
    segments = [map(int, s) for s in segments]
    return segments
1215655
def inflate_dtype(arr, names):
    """"""
    arr = np.asanyarray(arr)
    if has_structured_dt(arr):
        return arr.dtype
    s_dt = arr.dtype
    dt = [(n, s_dt) for n in names]
    dt = np.dtype(dt)
    return dt
933333
def read_mm_header(fd, byte_order, dtype, count):
    """"""
    return numpy.rec.fromfile(fd, MM_HEADER, 1, byteorder=byte_order)[0]
46026
def _clean(value):
    
    if isinstance(value, np.ndarray):
        if value.dtype.kind == 'S':
            return np.char.decode(value).tolist()
        else:
            return value.tolist()
    elif type(value).__module__ == np.__name__:
        # h5py==2.8.0 on windows sometimes fails to cast this from an np.float64 to a python.float
        # We have explicitly cast in Albacore (merge 488) to avoid this bug, since casting here could be dangerous
        # https://github.com/h5py/h5py/issues/1051
        conversion = np.asscalar(value)
        if sys.version_info.major == 3 and isinstance(conversion, bytes):
            conversion = conversion.decode()
        return conversion
    elif sys.version_info.major == 3 and isinstance(value, bytes):
        return value.decode()
    else:
        return value
1363649
def create_from_fits(cls, fitsfile, norm_type='flux'):
        
        tsmap = WcsNDMap.read(fitsfile)

        tab_e = Table.read(fitsfile, 'EBOUNDS')
        tab_s = Table.read(fitsfile, 'SCANDATA')
        tab_f = Table.read(fitsfile, 'FITDATA')

        tab_e = convert_sed_cols(tab_e)
        tab_s = convert_sed_cols(tab_s)
        tab_f = convert_sed_cols(tab_f)

        emin = np.array(tab_e['e_min'])
        emax = np.array(tab_e['e_max'])
        try:
            if str(tab_e['e_min'].unit) == 'keV':
                emin /= 1000.
        except:
            pass
        try:
            if str(tab_e['e_max'].unit) == 'keV':
                emax /= 1000.
        except:
            pass

        nebins = len(tab_e)
        npred = tab_e['ref_npred']

        ndim = len(tsmap.data.shape)

        if ndim == 2:
            cube_shape = (tsmap.data.shape[0],
                          tsmap.data.shape[1], nebins)
        elif ndim == 1:
            cube_shape = (tsmap.data.shape[0], nebins)
        else:
            raise RuntimeError("Counts map has dimension %i" % (ndim))

        refSpec = ReferenceSpec.create_from_table(tab_e)
        nll_vals = -np.array(tab_s["dloglike_scan"])
        norm_vals = np.array(tab_s["norm_scan"])

        axis = MapAxis.from_edges(np.concatenate((emin, emax[-1:])),
                                  interp='log')
        geom_3d = tsmap.geom.to_cube([axis])
        tscube = WcsNDMap(geom_3d,
                          np.rollaxis(tab_s["ts"].reshape(cube_shape), 2, 0))

        ncube = WcsNDMap(geom_3d,
                         np.rollaxis(tab_s["norm"].reshape(cube_shape), 2, 0))
        nmap = WcsNDMap(tsmap.geom,
                        tab_f['fit_norm'].reshape(tsmap.data.shape))

        ref_colname = 'ref_%s' % norm_type
        norm_vals *= tab_e[ref_colname][np.newaxis, :, np.newaxis]

        return cls(tsmap, nmap, tscube, ncube,
                   norm_vals, nll_vals, refSpec,
                   norm_type)
850972
def _load_fits(self, h5file):
        
        fits = {}
        for key in ['mf']:
            fits[key] = self._load_scalar_fit(fit_key=key, h5file=h5file)
        for key in ['chif', 'vf']:
            fits[key] = self._load_vector_fit(key, h5file)
        return fits
103658
def vid2h5(data: np.ndarray, ut1, rawind, ticks, outfn: Path, P: dict, i: int=0,
           Nfile: int=1, det=None, tstart=None, cmdlog: str=None):
    assert outfn, 'must provide a filename to write'

    outfn = Path(outfn).expanduser()
    assert not outfn.is_dir(
    ), 'must provide FILEname, you provided directory {outfn}'
    outfn.parent.mkdir(exist_ok=True)

    txtupd = f'convert file {i} / {Nfile}'
    if 'spoolfn' in P:
        txtupd += f' {P["spoolfn"].name}'
    txtupd += f' => {outfn}'
    
    print(txtupd + '\r', end="")
    # %% saving
    if outfn.suffix == '.h5':
        """
        Reference: https://www.hdfgroup.org/HDF5/doc/ADGuide/ImageSpec.html
        Thanks to Eric Piel of Delmic for pointing out this spec
        * the HDF5 attributess set are necessary to put HDFView into image mode and enables
        other conforming readers to easily play images stacks as video.
        * the string_() calls are necessary to make fixed length strings per HDF5 spec
        """
        # NOTE write mode r+ to not overwrite incremental images
        if outfn.is_file():
            writemode = 'r+'  # append
        else:
            writemode = 'w'

        if data is not None:

            N = data.shape[0] * Nfile
            ind = slice(i * data.shape[0], (i + 1) * data.shape[0])
        else:  # FIXME haven't reevaluated for update only case
            for q in (ut1, rawind, ticks):
                if q is not None:
                    N = len(q)
                    break
            ind = slice(None)

        with h5py.File(outfn, writemode) as f:
            if data is not None:
                if 'rawimg' not in f:  # first run
                    setupimgh5(f, N, data.shape[1], data.shape[2])

                f['/rawimg'][ind, ...] = data

            if ut1 is not None:
                print(f'writing from {datetime.utcfromtimestamp(ut1[0])}'
                      f'to {datetime.utcfromtimestamp(ut1[-1])}')
                if 'ut1_unix' not in f:
                    fut1 = f.create_dataset(
                        '/ut1_unix', shape=(N,), dtype=float, fletcher32=True)
                    fut1.attrs['units'] = 'seconds since Unix epoch Jan 1 1970 midnight'

                f['/ut1_unix'][ind] = ut1

            if tstart is not None and 'tstart' not in f:
                f['/tstart'] = tstart

            if rawind is not None:
                if 'rawind' not in f:
                    fri = f.create_dataset(
                        '/rawind', shape=(N,), dtype=np.int64, fletcher32=True)
                    fri.attrs['units'] = 'one-based index since camera program started this session'

                f['/rawind'][ind] = rawind

            if ticks is not None:
                if 'ticks' not in f:
                    ftk = f.create_dataset(
                        '/ticks', shape=(N,), dtype=np.uint64, fletcher32=True)  # Uint64
                    ftk.attrs['units'] = 'FPGA tick counter for each image frame'

                f['/ticks'][ind] = ticks

            if 'spoolfn' in P:
                # http://docs.h5py.org/en/latest/strings.html
                if 'spoolfn' not in f:
                    fsp = f.create_dataset(
                        '/spoolfn', shape=(N,), dtype=h5py.special_dtype(vlen=bytes))
                    fsp.attrs['description'] = 'input filename data was extracted from'

                f['/spoolfn'][ind] = P['spoolfn'].name

            if det is not None:
                if 'detect' not in f:
                    fdt = f.create_dataset('/detect', shape=(N,), dtype=int)
                    fdt.attrs['description'] = '# of auroral detections this frame'

                f['/detect'][i] = det[i]

            if 'params' not in f:
                cparam = np.array((
                    P['kineticsec'],
                    P['rotccw'],
                    P['transpose'],
                    P['flipud'],
                    P['fliplr'],
                    1),
                    dtype=[('kineticsec', 'f8'),
                           ('rotccw', 'i1'),
                           ('transpose', 'i1'),
                           ('flipud', 'i1'),
                           ('fliplr', 'i1'),
                           ('questionable_ut1', 'i1')]
                )

                # cannot use fletcher32 here, typeerror
                f.create_dataset('/params', data=cparam)

            if 'sensorloc' not in f and 'sensorloc' in P:
                loc = P['sensorloc']
                lparam = np.array((loc[0], loc[1], loc[2]),
                                  dtype=[('lat', 'f8'), ('lon', 'f8'), ('alt_m', 'f8')])

                # cannot use fletcher32 here, typeerror
                Ld = f.create_dataset('/sensorloc', data=lparam)
                Ld.attrs['units'] = 'WGS-84 lat (deg),lon (deg), altitude (meters)'

            if 'cmdlog' not in f:
                if isinstance(cmdlog, (tuple, list)):
                    cmdlog = ' '.join(cmdlog)
                # cannot use fletcher32 here, typeerror
                f['/cmdlog'] = str(cmdlog)

            if 'header' not in f and 'header' in P:
                f['/header'] = str(P['header'])

            if 'hdf5version' not in f:
                f['/hdf5version'] = h5py.version.hdf5_version_tuple

    elif outfn.suffix == '.fits':
        from astropy.io import fits
        # NOTE the with... syntax does NOT yet work with astropy.io.fits
        hdu = fits.PrimaryHDU(data)
        hdu.writeto(outfn, clobber=False, checksum=True)
        # no close
        """
        Note: the orientation of this FITS in NASA FV program and the preview
        image shown in Python should/must have the same orientation and pixel indexing')
        """

    elif outfn.suffix == '.mat':
        from scipy.io import savemat
        # matlab is fortran order
        matdata = {'imgdata': data.transpose(1, 2, 0)}
        savemat(outfn, matdata, oned_as='column')
    else:
        raise ValueError(f'what kind of file is {outfn}')
302948
def convert_tscube_old(infile, outfile):
    """"""
    inhdulist = fits.open(infile)

    # If already in the new-style format just write and exit
    if 'DLOGLIKE_SCAN' in inhdulist['SCANDATA'].columns.names:
        if infile != outfile:
            inhdulist.writeto(outfile, clobber=True)
        return

    # Get stuff out of the input file
    nrows = inhdulist['SCANDATA']._nrows
    nebins = inhdulist['EBOUNDS']._nrows
    npts = inhdulist['SCANDATA'].data.field('NORMSCAN').shape[1] / nebins

    emin = inhdulist['EBOUNDS'].data.field('e_min') / 1E3
    emax = inhdulist['EBOUNDS'].data.field('e_max') / 1E3
    eref = np.sqrt(emin * emax)
    dnde_emin = inhdulist['EBOUNDS'].data.field('E_MIN_FL')
    dnde_emax = inhdulist['EBOUNDS'].data.field('E_MAX_FL')
    index = np.log(dnde_emin / dnde_emax) / np.log(emin / emax)

    flux = PowerLaw.eval_flux(emin, emax, [dnde_emin, index], emin)
    eflux = PowerLaw.eval_eflux(emin, emax, [dnde_emin, index], emin)
    dnde = PowerLaw.eval_dnde(np.sqrt(emin * emax), [dnde_emin, index], emin)

    ts_map = inhdulist['PRIMARY'].data.reshape((nrows))
    ok_map = inhdulist['TSMAP_OK'].data.reshape((nrows))
    n_map = inhdulist['N_MAP'].data.reshape((nrows))
    errp_map = inhdulist['ERRP_MAP'].data.reshape((nrows))
    errn_map = inhdulist['ERRN_MAP'].data.reshape((nrows))
    err_map = np.ndarray((nrows))
    m = errn_map > 0
    err_map[m] = 0.5 * (errp_map[m] + errn_map[m])
    err_map[~m] = errp_map[~m]
    ul_map = n_map + 2.0 * errp_map

    ncube = np.rollaxis(inhdulist['N_CUBE'].data,
                        0, 3).reshape((nrows, nebins))
    errpcube = np.rollaxis(
        inhdulist['ERRPCUBE'].data, 0, 3).reshape((nrows, nebins))
    errncube = np.rollaxis(
        inhdulist['ERRNCUBE'].data, 0, 3).reshape((nrows, nebins))
    tscube = np.rollaxis(inhdulist['TSCUBE'].data,
                         0, 3).reshape((nrows, nebins))
    nll_cube = np.rollaxis(
        inhdulist['NLL_CUBE'].data, 0, 3).reshape((nrows, nebins))
    ok_cube = np.rollaxis(
        inhdulist['TSCUBE_OK'].data, 0, 3).reshape((nrows, nebins))

    ul_cube = ncube + 2.0 * errpcube
    m = errncube > 0
    errcube = np.ndarray((nrows, nebins))
    errcube[m] = 0.5 * (errpcube[m] + errncube[m])
    errcube[~m] = errpcube[~m]

    norm_scan = inhdulist['SCANDATA'].data.field(
        'NORMSCAN').reshape((nrows, npts, nebins)).swapaxes(1, 2)
    nll_scan = inhdulist['SCANDATA'].data.field(
        'NLL_SCAN').reshape((nrows, npts, nebins)).swapaxes(1, 2)

    # Adjust the "EBOUNDS" hdu
    columns = inhdulist['EBOUNDS'].columns
    columns.add_col(fits.Column(name=str('e_ref'),
                                format='E', array=eref * 1E3,
                                unit='keV'))
    columns.add_col(fits.Column(name=str('ref_flux'),
                                format='D', array=flux,
                                unit='ph / (cm2 s)'))
    columns.add_col(fits.Column(name=str('ref_eflux'),
                                format='D', array=eflux,
                                unit='MeV / (cm2 s)'))
    columns.add_col(fits.Column(name=str('ref_dnde'),
                                format='D', array=dnde,
                                unit='ph / (MeV cm2 s)'))

    columns.change_name('E_MIN_FL', str('ref_dnde_e_min'))
    columns.change_unit('ref_dnde_e_min', 'ph / (MeV cm2 s)')
    columns.change_name('E_MAX_FL', str('ref_dnde_e_max'))
    columns.change_unit('ref_dnde_e_max', 'ph / (MeV cm2 s)')
    columns.change_name('NPRED', str('ref_npred'))

    hdu_e = fits.BinTableHDU.from_columns(columns, name='EBOUNDS')

    # Make the "FITDATA" hdu
    columns = fits.ColDefs([])

    columns.add_col(fits.Column(
        name=str('fit_ts'), format='E', array=ts_map))
    columns.add_col(fits.Column(
        name=str('fit_status'), format='E', array=ok_map))
    columns.add_col(fits.Column(
        name=str('fit_norm'), format='E', array=n_map))
    columns.add_col(fits.Column(
        name=str('fit_norm_err'), format='E', array=err_map))
    columns.add_col(fits.Column(
        name=str('fit_norm_errp'), format='E', array=errp_map))
    columns.add_col(fits.Column(
        name=str('fit_norm_errn'), format='E', array=errn_map))
    hdu_f = fits.BinTableHDU.from_columns(columns, name='FITDATA')

    # Make the "SCANDATA" hdu
    columns = fits.ColDefs([])

    columns.add_col(fits.Column(name=str('ts'),
                                format='%iE' % nebins, array=tscube,
                                dim=str('(%i)' % nebins)))

    columns.add_col(fits.Column(name=str('bin_status'),
                                format='%iE' % nebins, array=ok_cube,
                                dim=str('(%i)' % nebins)))

    columns.add_col(fits.Column(name=str('norm'),
                                format='%iE' % nebins, array=ncube,
                                dim=str('(%i)' % nebins)))

    columns.add_col(fits.Column(name=str('norm_ul'),
                                format='%iE' % nebins, array=ul_cube,
                                dim=str('(%i)' % nebins)))

    columns.add_col(fits.Column(name=str('norm_err'),
                                format='%iE' % nebins, array=errcube,
                                dim=str('(%i)' % nebins)))

    columns.add_col(fits.Column(name=str('norm_errp'),
                                format='%iE' % nebins, array=errpcube,
                                dim=str('(%i)' % nebins)))

    columns.add_col(fits.Column(name=str('norm_errn'),
                                format='%iE' % nebins, array=errncube,
                                dim=str('(%i)' % nebins)))

    columns.add_col(fits.Column(name=str('loglike'),
                                format='%iE' % nebins, array=nll_cube,
                                dim=str('(%i)' % nebins)))

    columns.add_col(fits.Column(name=str('norm_scan'),
                                format='%iE' % (nebins * npts),
                                array=norm_scan,
                                dim=str('(%i,%i)' % (npts, nebins))))

    columns.add_col(fits.Column(name=str('dloglike_scan'),
                                format='%iE' % (nebins * npts),
                                array=nll_scan,
                                dim=str('(%i,%i)' % (npts, nebins))))

    hdu_s = fits.BinTableHDU.from_columns(columns, name='SCANDATA')

    hdulist = fits.HDUList([inhdulist[0],
                            hdu_s,
                            hdu_f,
                            inhdulist["BASELINE"],
                            hdu_e])

    hdulist['SCANDATA'].header['UL_CONF'] = 0.95

    hdulist.writeto(outfile, clobber=True)

    return hdulist
210970
def arrays_to_hdf5(filename="cache.hdf5"):
    """"""
    return Registry(
        types={
            numpy.ndarray: SerNumpyArrayToHDF5(filename, "cache.lock")
        },
        hooks={
            '<ufunc>': SerUFunc()
        },
        hook_fn=_numpy_hook
    )
860858
def save_hdf5(X, y, path):
    

    with h5py.File(path, 'w') as f:
        is_sparse = 1 if sparse.issparse(X) else 0
        f['issparse'] = is_sparse
        f['target'] = y

        if is_sparse:
            if not sparse.isspmatrix_csr(X):
                X = X.tocsr()

            f['shape'] = np.array(X.shape)
            f['data'] = X.data
            f['indices'] = X.indices
            f['indptr'] = X.indptr
        else:
            f['data'] = X
1322971
def create_from_yamlfile(cls, yamlfile):
        """"""
        data = load_yaml(yamlfile)
        nebins = len(data)
        emin = np.array([data[i]['emin'] for i in range(nebins)])
        emax = np.array([data[i]['emax'] for i in range(nebins)])
        ref_flux = np.array([data[i]['flux'][1] for i in range(nebins)])
        ref_eflux = np.array([data[i]['eflux'][1] for i in range(nebins)])
        conv = np.array([data[i]['eflux2npred'] for i in range(nebins)])
        ref_npred = conv*ref_eflux
        ones = np.ones(ref_flux.shape)
        ref_spec = ReferenceSpec(emin, emax, ones, ref_flux, ref_eflux, ref_npred)
        norm_data = np.array([data[i]['eflux'] for i in range(nebins)])
        ll_data =  np.array([data[i]['logLike'] for i in range(nebins)])
        max_ll = ll_data.max(1)
        nll_data = (max_ll - ll_data.T).T
        return cls(norm_data, nll_data, ref_spec, 'eflux')
248541
def parse_args():
    """"""
    parser = argparse.ArgumentParser(
        description='''Convert a set of files to a single h5features file, input files
        can be npz (numpy) or mat (MATLAB) files.''')

    parser.add_argument('file', nargs='+', type=str,
                        help='File to convert in the h5features format')

    parser.add_argument('-o', '--output', default='features.h5', type=str,
                        help='''The output h5features file to write on
                        (default is %(default)s)''')

    parser.add_argument('-g', '--group', default='h5features', type=str,
                        help='''The group to write in the output file
                        (default is %(default)s)''')

    parser.add_argument('--chunk', default=0.1, type=float,
                        help='''size of a file chunk in MB
                        (default is %(default)s)''')

    return parser.parse_args()