Creates the numpy array tables from the hdf5 tables
430800
def _build_data(self, amplification_group):
        
        # Determine shape of the tables
        n_levels = len(amplification_group)
        # Checks the first group in the amplification group and returns the
        # shape of the SA array - implicitly assumes the SA array in all
        # amplification groups is the same shape
        level = next(iter(amplification_group))
        n_d, n_p, n_m = amplification_group[level]["IMLs/SA"].shape
        assert n_d == len(self.distances), (n_d, len(self.distances))
        assert n_m == len(self.magnitudes), (n_m, len(self.magnitudes))
        # Instantiate the arrays with ones
        self.mean = {"SA": numpy.ones([n_d, n_p, n_m, n_levels]),
                     "PGA": numpy.ones([n_d, 1, n_m, n_levels]),
                     "PGV": numpy.ones([n_d, 1, n_m, n_levels])}
        self.sigma = {}
        for stddev_type in [const.StdDev.TOTAL, const.StdDev.INTER_EVENT,
                            const.StdDev.INTRA_EVENT]:
            level = next(iter(amplification_group))
            if stddev_type in amplification_group[level]:
                self.sigma[stddev_type] = deepcopy(self.mean)

        for iloc, (level, amp_model) in enumerate(amplification_group.items()):
            if "SA" in amp_model["IMLs"]:
                if iloc == 0:
                    self.periods = amp_model["IMLs/T"][:]
                else:
                    assert numpy.allclose(self.periods, amp_model["IMLs/T"][:])
            for imt in ["SA", "PGA", "PGV"]:
                if imt in amp_model["IMLs"]:
                    self.mean[imt][:, :, :, self.argidx[iloc]] = \
                        amp_model["IMLs/" + imt][:]
                    for stddev_type in self.sigma:
                        self.sigma[stddev_type][imt][
                            :, :, :, self.argidx[iloc]] = \
                            amp_model["/".join([stddev_type, imt])][:]
        self.shape = (n_d, n_p, n_m, n_levels)
737932
def to_table(data, headers)
      column_lengths = []
      gutter = 2
      table = ''

      # Figure out column widths based on longest string in each column (including the header string)
      headers.each { |header|
        width = data.map { |_| _[header] }.max_by(&:length).length

        width = header.length if width < header.length

        column_lengths << width
      }

      # format the length of a table cell string to make it as wide as the column (by adding extra spaces)
      format_row_entry = lambda { |entry, i|
        entry + (' ' * (column_lengths[i] - entry.length + gutter))
      }

      # Add headers
      headers.each_with_index { |header, i|
        table += format_row_entry.call(header, i)
      }

      table += "\n"

      # Add line breaks under headers
      column_lengths.each { |length|
        table += (('-' * length) + (' ' * gutter))
      }

      table += "\n"

      # Add rows
      data.each { |row|
        headers.each_with_index { |header, i|
          table += format_row_entry.call(row[header], i)
        }

        table += "\n"
      }

      table
    end
1028206
def write_tables_to_fits(filepath, tablelist, clobber=False,
                         namelist=None, cardslist=None, hdu_list=None):
    
    outhdulist = [fits.PrimaryHDU()]
    rmlist = []
    for i, table in enumerate(tablelist):
        ft_name = "%s._%i" % (filepath, i)
        rmlist.append(ft_name)
        try:
            os.unlink(ft_name)
        except:
            pass
        table.write(ft_name, format="fits")
        ft_in = fits.open(ft_name)
        if namelist:
            ft_in[1].name = namelist[i]
        if cardslist:
            for k, v in cardslist[i].items():
                ft_in[1].header[k] = v
        ft_in[1].update()
        outhdulist += [ft_in[1]]

    if hdu_list is not None:
        for h in hdu_list:
            outhdulist.append(h)

    fits.HDUList(outhdulist).writeto(filepath, overwrite=clobber)
    for rm in rmlist:
        os.unlink(rm)
642532
def dumpDictHdf5(RV,o):
    
    for key in list(RV.keys()):
        o.create_dataset(name=key,data=SP.array(RV[key]),chunks=True,compression='gzip')
417212
def make_table(self):
        """"""
        num_records = np.sum([1 for frame in self.timeseries])
        dtype = [("frame",float),("time",float),("ligand atom id",int),
                ("ligand atom name","|U4"),("cutoff",float),
                ("protein atom names",list),("protein atom ids",list),
                ("resid",int),("resname","|U4"),("segid","|U8") ]
        out = np.empty((num_records,),dtype=dtype)
        cursor=0
        for contact in self.timeseries:
            out[cursor] = (contact.frame, contact.time,contact.ligandatomindex,contact.ligandatomname,contact.cutoff,
                           contact.proteinatomname,contact.proteinatomindex,contact.resid,contact.resname,contact.segid)
            cursor+=1
        return out.view(np.recarray)
522662
def hdf5_col(self, chain=-1):
        
        return self.db._tables[chain].colinstances[self.name]
404178
def loadhdf(filename, groupname="data", to_dataframe=False):
    
    data = {}
    with _h5py.File(filename, "r") as f:
        for key, value in f[groupname].items():
            data[key] = np.array(value)
    if to_dataframe:
        return _pd.DataFrame(data)
    else:
        return data
811475
def _make_repr_table_from_sframe(X):
    

    assert isinstance(X, _SFrame)

    column_names = X.column_names()

    out_data = [ [None]*len(column_names) for i in range(X.num_rows())]

    column_sizes = [len(s) for s in column_names]

    for i, c in enumerate(column_names):
        for j, e in enumerate(X[c]):
            out_data[j][i] = str(e)
            column_sizes[i] = max(column_sizes[i], len(e))

    # now, go through and pad everything.
    out_data = ([ [cn.ljust(k, ' ') for cn, k in zip(column_names, column_sizes)],
                  ["-"*k for k in column_sizes] ]
                + [ [e.ljust(k, ' ') for e, k in zip(row, column_sizes)] for row in out_data] )

    return ['  '.join(row) for row in out_data]
491890
def _from_numpy(np_array):
    

    # Convert the numpy array to nd4j context dtype
    required_dtype = get_np_dtype(get_context_dtype())
    if np_array.dtype != required_dtype:
        raise Exception("{} is required, got {}".format(
            repr(required_dtype), repr(np_array.dtype)))

    # Nd4j does not have 1-d vectors.
    # So we add a dummy dimension.
    if np_array.ndim == 1:
        np_array = np.expand_dims(np_array, 0)

    # We have to maintain references to all incoming
    # numpy arrays. Else they will get GCed

    # creates a Nd4j array from a numpy array
    # To create an Nd4j array, we need 3 things:
    # buffer, strides, and shape

    # Get the buffer
    # A buffer is basically an array. To get the buffer object
    # we need a pointer to the first element and the size.
    pointer_address, _ = np_array.__array_interface__['data']
    _refs.append(np_array)
    pointer = native_ops.pointerForAddress(pointer_address)
    size = np_array.size
    mapping = {
        np.float64: DoublePointer,
        np.float32: FloatPointer,
    }
    pointer = mapping[required_dtype](pointer)
    buff = Nd4j.createBuffer(pointer, size)
    assert buff.address() == pointer_address
    _refs.append(buff)
    # Get the strides
    # strides = tuple of bytes to step in each
    # dimension when traversing an array.
    elem_size = buff.getElementSize()
    # Make sure word size is same in both python
    # and java worlds
    assert elem_size == np_array.dtype.itemsize
    strides = np_array.strides
    # numpy uses byte wise strides. We have to
    # convert it to word wise strides.
    strides = [dim / elem_size for dim in strides]

    # Finally, shape:
    shape = np_array.shape

    nd4j_array = Nd4j.create(buff, shape, strides, 0)
    assert buff.address() == nd4j_array.data().address()
    return nd4j_array
381704
def array(arr, *args, **kwargs):
    '''
    
    '''
    return weldarray(np.array(arr, *args, **kwargs))
1109473
def make_table(self):
        """"""
        num_records = int(np.sum([1 for frame in self.timeseries]))
        dtype = [
                ("frame",float),("time",float),("proteinring",list),
                ("ligand_ring_ids",list),("distance",float),("angle",float),
                ("offset",float),("type","|U4"),("resid",int),("resname","|U4"),("segid","|U8") ]
        out = np.empty((num_records,),dtype=dtype)
        cursor=0
        for contact in self.timeseries:
            out[cursor] = (contact.frame, contact.time,contact.proteinring,contact.ligandring,contact.distance,contact.angle,contact.offset,contact.type,contact.resid,contact.resname,contact.segid)
            cursor+=1
        return out.view(np.recarray)
999038
def inh(table):
    
    t = []
    for i in table:
        t.append(np.ndarray.tolist(np.arcsinh(i)))
    return t
217398
def write_data_chunk(self, rows, indexes, mask, values):
        

        # 0 len
        for v in values:
            if not np.prod(v.shape):
                return

        try:
            nrows = indexes[0].shape[0]
            if nrows != len(rows):
                rows = np.empty(nrows, dtype=self.dtype)
            names = self.dtype.names
            nindexes = len(indexes)

            # indexes
            for i, idx in enumerate(indexes):
                rows[names[i]] = idx

            # values
            for i, v in enumerate(values):
                rows[names[i + nindexes]] = v

            # mask
            if mask is not None:
                m = ~mask.ravel().astype(bool, copy=False)
                if not m.all():
                    rows = rows[m]

        except Exception as detail:
            raise Exception(
                "cannot create row-data -> {detail}".format(detail=detail))

        try:
            if len(rows):
                self.table.append(rows)
                self.table.flush()
        except Exception as detail:
            raise TypeError(
                "tables cannot write this data -> {detail}".format(
                    detail=detail))
1368251
def setupimgh5(f: Union[Path, h5py.File],
               Nframetotal: int, Nrow: int, Ncol: int, dtype=np.uint16,
               writemode='r+', key='/rawimg', cmdlog: str=None):
    
    if isinstance(f, (str, Path)):  # assume new HDF5 file wanted
        f = Path(f).expanduser()
        if f.is_dir():
            raise IOError(
                'must provide specific HDF5 filename, not just a directory')

        print('creating', f)
        with h5py.File(f, writemode) as F:
            setupimgh5(F, Nframetotal, Nrow, Ncol, dtype, writemode, key)

    elif isinstance(f, h5py.File):
        h = f.create_dataset(key,
                             shape=(Nframetotal, Nrow, Ncol),
                             dtype=dtype,
                             chunks=(1, Nrow, Ncol),  # each image is a chunk
                             compression='gzip',
                             # no difference in filesize from 1 to 5, except much faster to use lower numbers!
                             compression_opts=1,
                             shuffle=True,
                             fletcher32=True,
                             track_times=True)
        h.attrs["CLASS"] = np.string_("IMAGE")
        h.attrs["IMAGE_VERSION"] = np.string_("1.2")
        h.attrs["IMAGE_SUBCLASS"] = np.string_("IMAGE_GRAYSCALE")
        h.attrs["DISPLAY_ORIGIN"] = np.string_("LL")
        h.attrs['IMAGE_WHITE_IS_ZERO'] = np.uint8(0)

        if cmdlog and isinstance(cmdlog, str):
            f['/cmdlog'] = cmdlog
    else:
        raise TypeError(
            f'{type(f)} is not correct, must be filename or h5py.File HDF5 file handle')
1242730
def table_from_root(source, treename=None, columns=None, **kwargs):
    
    import root_numpy

    # parse column filters into tree2array ``selection`` keyword
    # NOTE: not all filters can be passed directly to root_numpy, so we store
    #       those separately and apply them after-the-fact before returning
    try:
        selection = kwargs.pop('selection')
    except KeyError:  # no filters
        filters = None
    else:
        rootfilters = []
        filters = []
        for col, op_, value in parse_column_filters(selection):
            try:
                opstr = [key for key in OPERATORS if OPERATORS[key] is op_][0]
            except (IndexError, KeyError):  # cannot filter with root_numpy
                filters.append((col, op_, value))
            else:  # can filter with root_numpy
                rootfilters.append('{0} {1} {2!r}'.format(col, opstr, value))
        kwargs['selection'] = ' && '.join(rootfilters)

    # pass file name (not path)
    if not isinstance(source, string_types):
        source = source.name

    # find single tree (if only one tree present)
    if treename is None:
        trees = root_numpy.list_trees(source)
        if len(trees) == 1:
            treename = trees[0]
        elif not trees:
            raise ValueError("No trees found in %s" % source)
        else:
            raise ValueError("Multiple trees found in %s, please select on "
                             "via the `treename` keyword argument, e.g. "
                             "`treename='events'`. Available trees are: %s."
                             % (source, ', '.join(map(repr, trees))))

    # read, filter, and return
    t = Table(root_numpy.root2array(
        source,
        treename,
        branches=columns,
        **kwargs
    ))
    if filters:
        return filter_table(t, *filters)
    return t
791538
def register_hdf5_array_io(array_type, format='hdf5', identify=True):
    
    def from_hdf5(*args, **kwargs):
        """Read an array from HDF5
        """
        kwargs.setdefault('array_type', array_type)
        return read_hdf5_array(*args, **kwargs)

    io_registry.register_reader(format, array_type, from_hdf5)
    if issubclass(array_type, Series):
        io_registry.register_writer(format, array_type, write_hdf5_series)
    else:
        io_registry.register_writer(format, array_type, write_hdf5_array)

    if identify:
        io_registry.register_identifier(format, array_type,
                                        io_hdf5.identify_hdf5)
966395
def create_table(self, name, data, columns=None, add_pk=True):
        """"""
        # TODO: Issue occurs when bool values exist in data
        # Remove if the table exists
        if name in self.tables:
            self.drop(name)

        # Set headers list
        if not columns:
            columns = data[0]

        # Validate data shape
        for row in data:
            assert len(row) == len(columns)

        # Create dictionary of column types
        col_types = {columns[i]: sql_column_type([d[i] for d in data], prefer_int=True, prefer_varchar=True)
                     for i in range(0, len(columns))}

        # Join column types into SQL string
        cols = ''.join(['\t{0} {1},\n'.format(name, type_) for name, type_ in col_types.items()])[:-2] + '\n'
        statement = 'CREATE TABLE {0} ({1}{2})'.format(name, '\n', cols)
        self.execute(statement)
        if add_pk:
            self.set_primary_key_auto()
        return True