Reads the standard deviation tables from hdf5 and stores them in
        memory
        :param fle:
            HDF5 Tables as instance of :class:`h5py.File`
189035
def _setup_standard_deviations(self, fle):
        
        # Load in total standard deviation
        self.stddevs = {}
        self.stddevs[const.StdDev.TOTAL] = hdf_arrays_to_dict(fle["Total"])
        # If other standard deviations
        self.DEFINED_FOR_STANDARD_DEVIATION_TYPES = set(
            self.DEFINED_FOR_STANDARD_DEVIATION_TYPES)
        for stddev_type in [const.StdDev.INTER_EVENT,
                            const.StdDev.INTRA_EVENT]:
            if stddev_type in fle:
                self.stddevs[stddev_type] = hdf_arrays_to_dict(
                    fle[stddev_type])
                self.DEFINED_FOR_STANDARD_DEVIATION_TYPES.add(stddev_type)
130403
def _get_stddevs(self, C, stddev_types, nsites):
        
        stddevs = []
        for stddev_type in stddev_types:
            assert stddev_type in self.DEFINED_FOR_STANDARD_DEVIATION_TYPES
            if stddev_type == const.StdDev.TOTAL:
                stddevs.append(C['sigma'] + np.zeros(nsites, dtype=float))
        return stddevs
1250804
def read_tables(fstream):
    
    table = read_table(fstream)
    while table is not None:
        yield table
        table = read_table(fstream)
237846
def _fromfile(self, parent):
        """"""
        fd = parent._fd
        byte_order = parent.byte_order

        self._offset = fd.tell()
        self.value_offset = self._offset + parent.offset_size + 4

        fmt, size = {4: ('HHI4s', 12), 8: ('HHQ8s', 20)}[parent.offset_size]
        data = fd.read(size)
        code, dtype = struct.unpack(byte_order + fmt[:2], data[:4])
        count, value = struct.unpack(byte_order + fmt[2:], data[4:])

        if code in TIFF_TAGS:
            name = TIFF_TAGS[code][0]
        elif code in CUSTOM_TAGS:
            name = CUSTOM_TAGS[code][0]
        else:
            name = str(code)

        try:
            dtype = TIFF_DATA_TYPES[dtype]
        except KeyError:
            raise ValueError("unknown TIFF tag data type %i" % dtype)

        fmt = '%s%i%s' % (byte_order, count*int(dtype[0]), dtype[1])
        size = struct.calcsize(fmt)
        if size > parent.offset_size or code in CUSTOM_TAGS:
            pos = fd.tell()
            tof = {4: 'I', 8: 'Q'}[parent.offset_size]
            self.value_offset = struct.unpack(byte_order+tof, value)[0]
            fd.seek(self.value_offset)
            if code in CUSTOM_TAGS:
                readfunc = CUSTOM_TAGS[code][1]
                value = readfunc(fd, byte_order, dtype, count)
                fd.seek(0, 2)  # bug in numpy/Python 3.x ?
                if isinstance(value, dict):  # numpy.core.records.record
                    value = Record(value)
            elif code in TIFF_TAGS or dtype[-1] == 's':
                value = struct.unpack(fmt, fd.read(size))
            else:
                value = read_numpy(fd, byte_order, dtype, count)
                fd.seek(0, 2)  # bug in numpy/Python 3.x ?
            fd.seek(pos)
        else:
            value = struct.unpack(fmt, value[:size])

        if not code in CUSTOM_TAGS:
            if len(value) == 1:
                value = value[0]

        if dtype.endswith('s'):
            value = stripnull(value)

        self.code = code
        self.name = name
        self.dtype = dtype
        self.count = count
        self.value = value
1120964
def read_hdf5_dict(h5f, names=None, group=None, **kwargs):
    
    # find group from which to read
    if group:
        h5g = h5f[group]
    else:
        h5g = h5f

    # find list of names to read
    if names is None:
        names = [key for key in h5g if _is_timeseries_dataset(h5g[key])]

    # read names
    out = kwargs.pop('dict_type', TimeSeriesDict)()
    kwargs.setdefault('array_type', out.EntryClass)
    for name in names:
        out[name] = read_hdf5_timeseries(h5g[name], **kwargs)

    return out
812092
def get_nts(self):
        """"""
        data = []
        nt_obj = None
        with open(self.fin) as fin_stream:
            for lnum, line in enumerate(fin_stream, 1):
                try:
                    line = line.rstrip('\r\n') # chomp
                    # Obtain Data if headers have been collected from the first line
                    if nt_obj is not None:
                        flds = re.split(self.sep, line)
                        self.convert_ints_floats(flds)
                        flds[6] = [s.strip() for s in flds[6].split(',')]
                        ntdata = nt_obj._make(flds)
                        data.append(ntdata)
                    # Obtain the header
                    else:
                        nt_obj = self._init_nt_hdr(line)
                except RuntimeError:
                    # Print headers
                    #if nt_obj is not None:
                    #  sys.stdout.write("{HDRS}\n".format(HDRS='\n'.join(nt_obj._fields)))
                    flds = re.split(self.sep, line)
                    print(len(flds), "FIELDS")
                    print(flds)
                    #raise Exception("{FIN}({LNUM}): {LINE}\n".format(
                    #  FIN=self.fin, LNUM=lnum, LINE=line))
                    # JUST SKIP LINES WITH INCOMPLETE DATA, BUT PRINT ERROR MESSAGE
                    sys.stdout.write("**ERROR: {FIN}({LNUM}): {LINE}\n".format(
                        FIN=self.fin, LNUM=lnum, LINE=line))
            if self.log is not None:
                self.log.write("  {:9} lines READ:  {}\n".format(len(data), self.fin))
        return data
495830
def with_read_hdf5(func):
    
    @wraps(func)
    def decorated_func(fobj, *args, **kwargs):
        # pylint: disable=missing-docstring
        if not isinstance(fobj, h5py.HLObject):
            if isinstance(fobj, FILE_LIKE):
                fobj = fobj.name
            with h5py.File(fobj, 'r') as h5f:
                return func(h5f, *args, **kwargs)
        return func(fobj, *args, **kwargs)

    return decorated_func
1367911
def _table_info(self, buf=None):
    
    metrics = [self.count().name('nrows')]
    for col in self.columns:
        metrics.append(self[col].count().name(col))

    metrics = self.aggregate(metrics).execute().loc[0]

    names = ['Column', '------'] + self.columns
    types = ['Type', '----'] + [repr(x) for x in self.schema().types]
    counts = ['Non-null #', '----------'] + [str(x) for x in metrics[1:]]
    col_metrics = util.adjoin(2, names, types, counts)
    result = 'Table rows: {}\n\n{}'.format(metrics[0], col_metrics)

    print(result, file=buf)
1016937
def to_df(self, method: str = 'MEMORY', **kwargs) -> 'pd.DataFrame':
        
        ll = self._is_valid()
        if ll:
            print(ll['LOG'])
            return None
        else:
            return self.sas.sasdata2dataframe(self.table, self.libref, self.dsopts, method, **kwargs)
1228834
def read_reg_file(f, foff=0x2D):
    
    # convenience function for reading in data
    def rd(st):
        return struct.unpack(st, f.read(struct.calcsize(st)))

    f.seek(0x19)
    if f.read(1) != b'A':
        # raise TypeError("Version of REG file is too new.")
        return {}

    f.seek(foff)
    nrecs = rd('<I')[0]  # TODO: should be '<H'
    rec_tab = [rd('<HHIII') for n in range(nrecs)]

    names = {}
    f.seek(foff + 20 * nrecs + 4)
    for r in rec_tab:
        d = f.read(r[2])
        if r[1] == 1539:  # '0306'
            # this is part of the linked list too, but contains a
            # reference to a table
            cd = struct.unpack('<HIII21sI', d)
            names[cd[5]] = cd[4].decode('iso8859').strip('\x00')
            # except:
            #     pass
        elif r[1] == 32769 or r[1] == 32771:  # b'0180' or b'0380'
            names[r[4]] = d[:-1].decode('iso8859')
        elif r[1] == 32774:  # b'0680'
            # this is a string that is referenced elsewhere (in a table)
            names[r[4]] = d[2:-1].decode('iso8859')
        elif r[1] == 32770:  # b'0280'
            # this is just a flattened numeric array
            names[r[4]] = np.frombuffer(d, dtype=np.uint32, offset=4)

    data = {}
    f.seek(foff + 20 * nrecs + 4)
    for r in rec_tab:
        d = f.read(r[2])
        if r[1] == 1538:  # '0206'
            # this is part of a linked list
            if len(d) == 43:
                cd = struct.unpack('<HIII21sd', d)
                data[cd[4].decode('iso8859').strip('\x00')] = cd[5]
            else:
                pass
        elif r[1] == 1537:  # b'0106'
            # name of property
            n = d[14:30].split(b'\x00')[0].decode('iso8859')
            # with value from names
            data[n] = names.get(struct.unpack('<I', d[35:39])[0], '')
        elif r[1] == 1793:  # b'0107'
            # this is a table of values
            nrow = struct.unpack('<H', d[4:6])[0]
            ncol = struct.unpack('<H', d[16:18])[0]
            if ncol != 0:
                cols = [struct.unpack('<16sHHHHHI', d[20 + 30 * i:50 + 30 * i])
                        for i in range(ncol)]
                colnames = [c[0].split(b'\x00')[0].decode('iso8859')
                            for c in cols]
                # TODO: type 2 is not a constant size? 31, 17
                rty2sty = {1: 'H', 3: 'I', 4: 'f', 5: 'H',
                           7: 'H', 8: 'd', 11: 'H', 12: 'H',
                           13: 'I', 14: 'I', 16: 'H'}
                coltype = '<' + ''.join([rty2sty.get(c[3], str(c[2]) + 's')
                                         for c in cols])
                lencol = struct.calcsize(coltype)
                tab = []
                for i in reversed(range(2, nrow + 2)):
                    rawrow = struct.unpack(coltype,
                                           d[-i * lencol: (1 - i) * lencol])
                    row = []
                    for j, p in enumerate(rawrow):
                        if cols[j][3] == 3:
                            row.append(names.get(p, str(p)))
                        else:
                            row.append(p)
                    tab.append(row)
                data[names[r[4]]] = [colnames, tab]
        elif r[1] == 1281 or r[1] == 1283:  # b'0105' or b'0305'
            fm = '<HHBIIhIdII12shIddQQB8sII12shIddQQB8s'
            m = struct.unpack(fm, d)
            nrecs = m[4]  # number of points in table

            # x_units = names.get(m[8], '')
            x_arr = m[14] * names.get(m[9], np.arange(nrecs - 1))
            y_arr = m[25] * names.get(m[20])
            y_units = names.get(m[19], '')
            if y_units == 'bar':
                y_arr *= 0.1  # convert to MPa
            # TODO: what to call this?
            data['Trace'] = Trace(y_arr, x_arr, name='')
        # elif r[1] == 1025:  # b'0104'
        #     # lots of zeros? maybe one or two numbers?
        #     # only found in REG entries that have long 0280 records
        #     fm = '<HQQQIHHHHIIHB'
        #     m = struct.unpack(fm, d)
        #     print(m)
        #     #print(r[1], len(d), binascii.hexlify(d))
        #     pass
        # elif r[1] == 512:  # b'0002'
        #     # either points to two null pointers or two other pointers
        #     # (indicates start of linked list?)
        #     print(r[1], len(d), binascii.hexlify(d))
        # elif r[1] == 769 or r[1] == 772:  # b'0103' or b'0403'
        #     # points to 2nd, 3rd & 4th records (two 0002 records and a 0180)
        #     b = binascii.hexlify
        #     print(b(d[10:14]), b(d[14:18]), b(d[18:22]))

    return data
1300738
def dataframe_setup(self):
        
        # Initialise a dictionary to store the sanitized headers and strings
        genesippr_dict = dict()
        # Try to open all the reports - use pandas to extract the results from any report that exists
        try:
            sippr_matrix = pd.read_csv(os.path.join(self.reportpath, 'genesippr.csv'),
                                       delimiter=',', index_col=0).T.to_dict()
        except FileNotFoundError:
            sippr_matrix = dict()
        try:
            conf_matrix = pd.read_csv(os.path.join(self.reportpath, 'confindr_report.csv'),
                                      delimiter=',', index_col=0).T.to_dict()
        except FileNotFoundError:
            conf_matrix = dict()
        try:
            gdcs_matrix = pd.read_csv(os.path.join(self.reportpath, 'GDCS.csv'),
                                      delimiter=',', index_col=0).T.to_dict()
        except FileNotFoundError:
            gdcs_matrix = dict()
        # Populate the header:sanitized string dictionary with results from all strains
        for sample in self.metadata:
            genesippr_dict[sample.name] = dict()
            try:
                genesippr_dict[sample.name]['eae'] = self.data_sanitise(sippr_matrix[sample.name]['eae'])
            except KeyError:
                genesippr_dict[sample.name]['eae'] = 0
            try:
                genesippr_dict[sample.name]['hlyAEc'] = self.data_sanitise(sippr_matrix[sample.name]['hlyAEc'])
            except KeyError:
                genesippr_dict[sample.name]['hlyAEc'] = 0
            try:
                genesippr_dict[sample.name]['VT1'] = self.data_sanitise(sippr_matrix[sample.name]['VT1'])
            except KeyError:
                genesippr_dict[sample.name]['VT1'] = 0
            try:
                genesippr_dict[sample.name]['VT2'] = self.data_sanitise(sippr_matrix[sample.name]['VT2'])
            except KeyError:
                genesippr_dict[sample.name]['VT2'] = 0
            try:
                genesippr_dict[sample.name]['hlyALm'] = self.data_sanitise(sippr_matrix[sample.name]['hlyALm'])
            except KeyError:
                genesippr_dict[sample.name]['hlyALm'] = 0
            try:
                genesippr_dict[sample.name]['IGS'] = self.data_sanitise(sippr_matrix[sample.name]['IGS'])
            except KeyError:
                genesippr_dict[sample.name]['IGS'] = 0
            try:
                genesippr_dict[sample.name]['inlJ'] = self.data_sanitise(sippr_matrix[sample.name]['inlJ'])
            except KeyError:
                genesippr_dict[sample.name]['inlJ'] = 0
            try:
                genesippr_dict[sample.name]['invA'] = self.data_sanitise(sippr_matrix[sample.name]['invA'])
            except KeyError:
                genesippr_dict[sample.name]['invA'] = 0
            try:
                genesippr_dict[sample.name]['stn'] = self.data_sanitise(sippr_matrix[sample.name]['stn'])
            except KeyError:
                genesippr_dict[sample.name]['stn'] = 0
            try:
                genesippr_dict[sample.name]['GDCS'] = self.data_sanitise(gdcs_matrix[sample.name]['Pass/Fail'],
                                                                         header='Pass/Fail')
            except KeyError:
                genesippr_dict[sample.name]['GDCS'] = 0
            try:
                genesippr_dict[sample.name]['Contamination'] = self.data_sanitise(
                    conf_matrix[sample.name]['ContamStatus'], header='ContamStatus')
            except KeyError:
                genesippr_dict[sample.name]['Contamination'] = 0
            try:
                genesippr_dict[sample.name]['Coverage'] = self.data_sanitise(
                    gdcs_matrix[sample.name]['MeanCoverage'], header='MeanCoverage')
            except KeyError:
                genesippr_dict[sample.name]['Coverage'] = 0
        # Create a report from the header: sanitized string dictionary to be used in the creation of the report image
        with open(self.image_report, 'w') as csv:
            data = '{}\n'.format(','.join(self.header_list))
            for strain in sorted(genesippr_dict):
                data += '{str},'.format(str=strain)
                for header in self.header_list[1:]:
                    data += '{value},'.format(value=genesippr_dict[strain][header])

                data = data.rstrip(',')
                data += '\n'
            csv.write(data)
291096
def load_hdf(cls, filename, path=''):
        
        df_long = pd.read_hdf(filename,'{}/long/df'.format(path))
        df_short = pd.read_hdf(filename,'{}/short/df'.format(path))
        return cls.from_df(df_long, df_short)
850972
def _load_fits(self, h5file):
        
        fits = {}
        for key in ['mf']:
            fits[key] = self._load_scalar_fit(fit_key=key, h5file=h5file)
        for key in ['chif', 'vf']:
            fits[key] = self._load_vector_fit(key, h5file)
        return fits
794314
def load_k40_coincidences_from_hdf5(filename, dom_id):
    

    with h5py.File(filename, 'r') as h5f:
        data = h5f['/k40counts/{0}'.format(dom_id)]
        livetime = data.attrs['livetime']
        data = np.array(data)

    return data, livetime
177830
def dat2hdf5(table_dir):
    

    import astropy.io.ascii as ascii
    import gzip
    from contextlib import closing

    readme_fname = os.path.join(table_dir, 'ReadMe')
    table_fname = os.path.join(table_dir, 'table1.dat.gz')
    h5_fname = os.path.join(table_dir, 'marshall.h5')

    # Extract the gzipped table
    with gzip.open(table_fname, 'rb') as f:
        # Read in the table using astropy's CDS table reader
        r = ascii.get_reader(ascii.Cds, readme=readme_fname)
        r.data.table_name = 'table1.dat' # Hack to deal with bug in CDS reader.
        table = r.read(f)
        print(table)

    # Reorder table entries according to Galactic (l, b)
    l = coordinates.Longitude(
        table['GLON'][:],
        wrap_angle=180.*units.deg)
    b = table['GLAT'][:]

    sort_idx = np.lexsort((b, l))

    l = l[sort_idx].astype('f4')
    b = b[sort_idx].astype('f4')
    l.shape = (801, 81)
    b.shape = (801, 81)

    # Extract arrays from the table
    chi2_all = np.reshape((table['x2all'][sort_idx]).astype('f4'), (801,81))
    chi2_giants = np.reshape((table['x2gts'][sort_idx]).astype('f4'), (801,81))

    A = np.empty((801*81,33), dtype='f4')
    sigma_A = np.empty((801*81,33), dtype='f4')
    dist = np.empty((801*81,33), dtype='f4')
    sigma_dist = np.empty((801*81,33), dtype='f4')

    for k in range(33):
        A[:,k] = table['ext{:d}'.format(k+1)][sort_idx]
        sigma_A[:,k] = table['e_ext{:d}'.format(k+1)][sort_idx]
        dist[:,k] = table['r{:d}'.format(k+1)][sort_idx]
        sigma_dist[:,k] = table['e_r{:d}'.format(k+1)][sort_idx]

    A.shape = (801,81,33)
    sigma_A.shape = (801,81,33)
    dist.shape = (801,81,33)
    sigma_dist.shape = (801,81,33)

    # Construct the HDF5 file
    h5_fname = os.path.join(table_dir, 'marshall.h5')
    filter_kwargs = dict(
        chunks=True,
        compression='gzip',
        compression_opts=3,
        # scaleoffset=4
    )

    with h5py.File(h5_fname, 'w') as f:
        dset = f.create_dataset('A', data=A, **filter_kwargs)
        dset.attrs['description'] = 'Extinction of each bin'
        dset.attrs['band'] = 'Ks (2MASS)'
        dset.attrs['units'] = 'mag'

        dset = f.create_dataset('sigma_A', data=sigma_A, **filter_kwargs)
        dset.attrs['description'] = 'Extinction uncertainty of each bin'
        dset.attrs['band'] = 'Ks (2MASS)'
        dset.attrs['units'] = 'mag'

        dset = f.create_dataset('dist', data=dist, **filter_kwargs)
        dset.attrs['description'] = 'Distance of each bin'
        dset.attrs['units'] = 'kpc'

        dset = f.create_dataset('sigma_dist', data=sigma_dist, **filter_kwargs)
        dset.attrs['description'] = 'Distance uncertainty of each bin'
        dset.attrs['units'] = 'kpc'

        dset = f.create_dataset('chi2_all', data=chi2_all, **filter_kwargs)
        dset.attrs['description'] = 'Chi^2, based on all the stars'
        dset.attrs['units'] = 'unitless'

        dset = f.create_dataset('chi2_giants', data=chi2_giants, **filter_kwargs)
        dset.attrs['description'] = 'Chi^2, based on giants only'
        dset.attrs['units'] = 'unitless'

        # filter_kwargs.pop('scaleoffset')

        dset = f.create_dataset('l', data=l, **filter_kwargs)
        dset.attrs['description'] = 'Galactic longitude'
        dset.attrs['units'] = 'deg'

        dset = f.create_dataset('b', data=b, **filter_kwargs)
        dset.attrs['description'] = 'Galactic latitude'
        dset.attrs['units'] = 'deg'
183904
def __read_seq_data(fh):
  
  line = None
  seq_data = ""
  while line is None or not _isSequenceHead(line):
    line = fh.readline()
    if line == "":
      break  # file is finished...
    if not _isSequenceHead(line):
      seq_data += line.strip()
  return seq_data, line
186555
def from_file(fn, **options):
        
        if hasattr(fn, 'read'):
            return TableFu(fn, **options)
        with open(fn) as f:
            return TableFu(f, **options)