Read a `DataQualityDict` from an HDF5 file
975505
def read_hdf5_dict(h5f, names=None, path=None, on_missing='error', **kwargs):
    
    if path:
        h5f = h5f[path]

    # allow alternative keyword argument name (FIXME)
    if names is None:
        names = kwargs.pop('flags', None)

    # try and get list of names automatically
    if names is None:
        try:
            names = find_flag_groups(h5f, strict=True)
        except KeyError:
            names = None
        if not names:
            raise ValueError("Failed to automatically parse available flag "
                             "names from HDF5, please give a list of names "
                             "to read via the ``names=`` keyword")

    # read data
    out = DataQualityDict()
    for name in names:
        try:
            out[name] = read_hdf5_flag(h5f, name, **kwargs)
        except KeyError as exc:
            if on_missing == 'ignore':
                pass
            elif on_missing == 'warn':
                warnings.warn(str(exc))
            else:
                raise ValueError('no H5Group found for flag '
                                 '{0!r}'.format(name))

    return out
968254
def hdf5_read_out(self):
        
        with h5py.File(self.WORKING_DIRECTORY + '/' + self.output_file_name, 'w') as f:

            header = f.create_group('header')
            header.attrs['Title'] = 'Generated SNR Out'
            header.attrs['Author'] = 'Generator by: Michael Katz'
            header.attrs['Date/Time'] = str(datetime.datetime.now())

            for which in ['x', 'y']:
                header.attrs[which + 'val_name'] = getattr(self, which + 'val_name')
                header.attrs['num_' + which + '_pts'] = getattr(self, 'num_' + which)

            ecc = 'eccentricity' in self.__dict__
            if ecc:
                name_list = ['observation_time', 'start_frequency', 'start_separation'
                             'eccentricity']
            else:
                name_list = ['spin_1', 'spin_2', 'spin', 'end_time']

            name_list += ['total_mass', 'mass_ratio', 'start_time', 'luminosity_distance',
                          'comoving_distance', 'redshift']

            for name in name_list:
                if name != self.xval_name and name != self.yval_name:
                    try:
                        getattr(self, name)
                        header.attrs[name] = getattr(self, name)
                    except AttributeError:
                        pass

            if self.added_note != '':
                header.attrs['Added note'] = self.added_note

            data = f.create_group('data')

            # read out x,y values in compressed data set
            dset = data.create_dataset(self.x_col_name, data=self.xvals,
                                       dtype='float64', chunks=True,
                                       compression='gzip', compression_opts=9)

            dset = data.create_dataset(self.y_col_name, data=self.yvals,
                                       dtype='float64', chunks=True,
                                       compression='gzip', compression_opts=9)

            # read out all datasets
            for key in self.output_dict.keys():
                dset = data.create_dataset(key, data=self.output_dict[key],
                                           dtype='float64', chunks=True,
                                           compression='gzip', compression_opts=9)
598940
def _load_from_disk(self, dtype_out_time, dtype_out_vert=False,
                        region=False):
        """"""
        ds = xr.open_dataset(self.path_out[dtype_out_time])
        if region:
            arr = ds[region.name]
            # Use region-specific pressure values if available.
            if (self.dtype_in_vert == internal_names.ETA_STR
                    and not dtype_out_vert):
                reg_pfull_str = region.name + '_pressure'
                arr = arr.drop([r for r in arr.coords.iterkeys()
                                if r not in (internal_names.PFULL_STR,
                                             reg_pfull_str)])
                # Rename pfull to pfull_ref always.
                arr = arr.rename({internal_names.PFULL_STR:
                                  internal_names.PFULL_STR + '_ref'})
                # Rename region_pfull to pfull if its there.
                if hasattr(arr, reg_pfull_str):
                    return arr.rename({reg_pfull_str:
                                       internal_names.PFULL_STR})
                return arr
            return arr
        return ds[self.name]
90797
def save_hdf_metadata(filename, metadata, groupname="data", mode="a"):
    """""""
    with _h5py.File(filename, mode) as f:
        for key, val in metadata.items():
            f[groupname].attrs[key] = val
834379
def read_hdf(filename: PathLike, key: str) -> AnnData:
    
    with h5py.File(filename, 'r') as f:
        # the following is necessary in Python 3, because only
        # a view and not a list is returned
        keys = [k for k in f.keys()]
        if key == '':
            raise ValueError((
                'The file {} stores the following sheets:\n{}\n'
                'Call read/read_hdf5 with one of them.'
            ).format(filename, keys))
        # read array
        X = f[key][()]
        # try to find row and column names
        rows_cols = [{}, {}]
        for iname, name in enumerate(['row_names', 'col_names']):
            if name in keys:
                rows_cols[iname][name] = f[name][()]
    adata = AnnData(X, rows_cols[0], rows_cols[1], dtype=X.dtype.name)
    return adata
1334760
def load(self):
        
        hdf_filename = os.path.join(self._dump_dirname, 'result.h5')
        if os.path.isfile(hdf_filename):
            store = pd.HDFStore(hdf_filename, mode='r')
            keys = store.keys()
            if keys == ['/df']:
                self.result = store['df']
            else:
                if set(keys) == set(map(lambda i: '/%s' % i, range(len(keys)))):
                    # keys are not necessarily ordered
                    self.result = [store[str(k)] for k in range(len(keys))]
                else:
                    self.result = {k[1:]: store[k] for k in keys}

        else:
            self.result = joblib.load(
                    os.path.join(self._output_dirname, 'dump', 'result.pkl'))
1197137
def wr_xlsx(self, fout_xlsx):
        """"""
        from goatools.wr_tbl import wr_xlsx
        kws = {
            'title' : self.title,
            'hdrs' :["Dep/Lev",
                     "BP Depth", "MF Depth", "CC Depth",
                     "BP Level", "MF Level", "CC Level"]}
        wr_xlsx(fout_xlsx, self.get_data(), **kws)
595821
def analyze_quality(self):
    """"""
    res = {}
    for h in self._hpas:
      if h.type() not in res: res[h.type()]={}
      for c in h.get_quality():
        if c not in res[h.type()]: res[h.type()][c] = 0
        res[h.type()][c]+=1
    self._quality_distro = res
185048
def read(self, filename):
        
        f = h5py.File(filename, "r")
        self.data = []
        for i in range(f['data'].attrs['length']):
            self.data.append(f['data']['data_' + str(i)].value)
        self.u = []
        for i in range(f['u'].attrs['length']):
            self.u.append(f['u']['u_' + str(i)].value)
        self.sigma = []
        for i in range(f['sigma'].attrs['length']):
            self.sigma.append(f['sigma']['sigma_' + str(i)].value)
        self.v = []
        for i in range(f['v'].attrs['length']):
            self.v.append(f['v']['v_' + str(i)].value)
        self.stachans = [tuple(stachan.decode('ascii').split('.'))
                         for stachan in f['stachans'].value]
        self.dimension = f['data'].attrs['dimension']
        self.filt_order = f['data'].attrs['filt_order']
        self.highcut = f['data'].attrs['highcut']
        self.lowcut = f['data'].attrs['lowcut']
        self.multiplex = bool(f['data'].attrs['multiplex'])
        self.sampling_rate = f['data'].attrs['sampling_rate']
        if isinstance(f['data'].attrs['name'], str):
            self.name = f['data'].attrs['name']
        else:
            self.name = f['data'].attrs['name'].decode('ascii')
        return self
867907
def compare_hdf_files(file1, file2):
    
    data1 = FileToDict()
    data2 = FileToDict()
    scanner1 = data1.scan
    scanner2 = data2.scan
    with h5py.File(file1, 'r') as fh1:
        fh1.visititems(scanner1)
    with h5py.File(file2, 'r') as fh2:
        fh2.visititems(scanner2)
    return data1.contents == data2.contents
914038
def _parse_dict(self, data):
		
		d = {}
		for line in data.split('\r\n'):
			match = GNTP_HEADER.match(line)
			if not match:
				continue

			key = match.group(1).strip()
			val = match.group(2).strip()
			d[key] = val
		return d
881835
def __ReadFromFile(self, datapath):
        
        def _create_meta(_datapath):
            """Just simply returns some dict. This functions exists in order to keep DRY"""
            meta = {
                'series_number': 0,
                'datadir': _datapath
            }
            return meta

        path, ext = os.path.splitext(datapath)
        ext = ext[1:]
        if ext in ('pklz', 'pkl'):
            logger.debug('pklz format detected')
            from . import misc
            data = misc.obj_from_file(datapath, filetype='pkl')
            data3d = data.pop('data3d')
            # metadata must have series_number
            metadata = _create_meta(datapath)
            metadata.update(data)

        elif ext in ['hdf5']:
            from . import hdf5_io
            datap = hdf5_io.load_dict_from_hdf5(datapath)
            # datap = self.read_hdf5(datapath)
            data3d = datap.pop('data3d')

            # back compatibility
            if 'metadata' in datap.keys():
                datap = datap['metadata']
            # metadata must have series_number
            metadata = _create_meta(datapath)
            metadata.update(datap)

        elif ext in ['idx']:
            from . import idxformat
            idxreader = idxformat.IDXReader()
            data3d, metadata = idxreader.read(datapath)
        elif ext in ['dcm', 'DCM', 'dicom']:
            data3d, metadata = self._read_with_sitk(datapath)
            metadata = self._fix_sitk_bug(datapath, metadata)
        elif ext in ["bz2"]:
            new_datapath = tgz.untar(datapath)
            data3d, metadata = self.__ReadFromDirectory(new_datapath)
        else:
            logger.debug('file format "' + str(ext) + '"')
            # reading raw file
            data3d, metadata = self._read_with_sitk(datapath)
        return data3d, metadata
167535
def _from_dict(cls, _dict):
        """"""
        args = {}
        if 'element_pair' in _dict:
            args['element_pair'] = [
                ElementPair._from_dict(x) for x in (_dict.get('element_pair'))
            ]
        if 'identical_text' in _dict:
            args['identical_text'] = _dict.get('identical_text')
        if 'provenance_ids' in _dict:
            args['provenance_ids'] = _dict.get('provenance_ids')
        if 'significant_elements' in _dict:
            args['significant_elements'] = _dict.get('significant_elements')
        return cls(**args)
683590
def read_hdf5_segmentlist(h5f, path=None, gpstype=LIGOTimeGPS, **kwargs):
    
    # find dataset
    dataset = io_hdf5.find_dataset(h5f, path=path)

    segtable = Table.read(dataset, format='hdf5', **kwargs)
    out = SegmentList()
    for row in segtable:
        start = LIGOTimeGPS(int(row['start_time']), int(row['start_time_ns']))
        end = LIGOTimeGPS(int(row['end_time']), int(row['end_time_ns']))
        if gpstype is LIGOTimeGPS:
            out.append(Segment(start, end))
        else:
            out.append(Segment(gpstype(start), gpstype(end)))
    return out
495830
def with_read_hdf5(func):
    
    @wraps(func)
    def decorated_func(fobj, *args, **kwargs):
        # pylint: disable=missing-docstring
        if not isinstance(fobj, h5py.HLObject):
            if isinstance(fobj, FILE_LIKE):
                fobj = fobj.name
            with h5py.File(fobj, 'r') as h5f:
                return func(h5f, *args, **kwargs)
        return func(fobj, *args, **kwargs)

    return decorated_func
800685
def hdf5(self):
        """"""
        if self._rundir['hdf5'] is UNDETERMINED:
            h5_folder = self.path / self.par['ioin']['hdf5_output_folder']
            if (h5_folder / 'Data.xmf').is_file():
                self._rundir['hdf5'] = h5_folder
            else:
                self._rundir['hdf5'] = None
        return self._rundir['hdf5']
953377
def _read(self, filename):
        """"""
        data = {}
        parsers = [
            FirstDataParser(), CoordinateParser(), EnergyGradParser(),
            SkipApproxHessian(), HessianParser(), MassParser(),
        ]
        with open(filename) as f:
            while True:
                line = f.readline()
                if line == "":
                    break
                # at each line, a parsers checks if it has to process a piece of
                # file. If that happens, the parser gets control over the file
                # and reads as many lines as it needs to collect data for some
                # attributes.
                for parser in parsers:
                    if parser.test(line, data):
                        parser.read(line, f, data)
                        break
        self.__dict__.update(data)