Convert ibis dtype to the pandas / numpy alternative
557606
def ibis_dtype_to_pandas(ibis_dtype):
    """"""
    assert isinstance(ibis_dtype, dt.DataType)

    if isinstance(ibis_dtype, dt.Timestamp) and ibis_dtype.timezone:
        return DatetimeTZDtype('ns', ibis_dtype.timezone)
    elif isinstance(ibis_dtype, dt.Interval):
        return np.dtype('timedelta64[{}]'.format(ibis_dtype.unit))
    elif isinstance(ibis_dtype, dt.Category):
        return CategoricalDtype()
    elif type(ibis_dtype) in _ibis_dtypes:
        return _ibis_dtypes[type(ibis_dtype)]
    else:
        return np.dtype(np.object_)
726322
def dtype(self, byte_order='='):
        '''
        

        '''
        return _np.dtype([(prop.name, prop.dtype(byte_order))
                          for prop in self.properties])
354642
def encode_df(df, data_types={}):
    

    numbers = []
    categories = []
    datetimes = []
        
    for column, series in df.iteritems():
        if column in data_types:
            dtypes = data_types[column]
        else:
            dtypes = {}
        
        if 'datetime' in dtypes:
            index = pd.DatetimeIndex(pd.to_datetime(series.values))
            df[column] = index.astype(np.int64).astype(float)
            datetimes.append(column)
            
        elif any([dtype in dtypes for dtype in ['category', 'label']]):
            # If this can be treated as a category, encode it
            df = pd.get_dummies(df,columns=[column])
            categories.append(column)
                    
        elif 'numeric' in dtypes:
            df[column] = clean_numbers(df[column].values)
            df[column] = df[column].astype(float)
            numbers.append(column)
            
        else:
            df.drop(column,1,inplace=True)

    # Scale continuous columns
    scaler = MinMaxScaler()
    df[numbers + datetimes] = scaler.fit_transform(df[numbers + datetimes])
    
    return df, categories, numbers, datetimes
757683
def restore_type(self, dtype, sample=None):
        

        # Pandas types
        if pdc.is_bool_dtype(dtype):
            return 'boolean'
        elif pdc.is_datetime64_any_dtype(dtype):
            return 'datetime'
        elif pdc.is_integer_dtype(dtype):
            return 'integer'
        elif pdc.is_numeric_dtype(dtype):
            return 'number'

        # Python types
        if sample is not None:
            if isinstance(sample, (list, tuple)):
                return 'array'
            elif isinstance(sample, datetime.date):
                return 'date'
            elif isinstance(sample, isodate.Duration):
                return 'duration'
            elif isinstance(sample, dict):
                return 'object'
            elif isinstance(sample, six.string_types):
                return 'string'
            elif isinstance(sample, datetime.time):
                return 'time'

        return 'string'
689062
def _get_tbl_numpy_dtype(self, colnum, include_endianness=True):
        
        table_type = self._info['hdutype']
        table_type_string = _hdu_type_map[table_type]
        try:
            ftype = self._info['colinfo'][colnum]['eqtype']
            if table_type == ASCII_TBL:
                npy_type = _table_fits2npy_ascii[abs(ftype)]
            else:
                npy_type = _table_fits2npy[abs(ftype)]
        except KeyError:
            raise KeyError("unsupported %s fits data "
                           "type: %d" % (table_type_string, ftype))

        istbit = False
        if (ftype == 1):
            istbit = True

        isvar = False
        if ftype < 0:
            isvar = True
        if include_endianness:
            # if binary we will read the big endian bytes directly,
            # if ascii we read into native byte order
            if table_type == ASCII_TBL:
                addstr = ''
            else:
                addstr = '>'
            if npy_type not in ['u1', 'i1', 'S', 'U']:
                npy_type = addstr+npy_type

        if npy_type == 'S':
            width = self._info['colinfo'][colnum]['width']
            npy_type = 'S%d' % width
        elif npy_type == 'U':
            width = self._info['colinfo'][colnum]['width']
            npy_type = 'U%d' % width

        return npy_type, isvar, istbit
743924
def _fix_tbit_dtype(self, array, colnums):
        
        descr = array.dtype.descr
        for i, colnum in enumerate(colnums):
            npy_type, isvar, istbit = self._get_tbl_numpy_dtype(colnum)
            if (istbit):
                coldescr = list(descr[i])
                coldescr[1] = '?'
                descr[i] = tuple(coldescr)

        return array.view(descr)
135118
def get_dtype_counts(self):
        
        if hasattr(self, "dtype"):
            return pandas.Series({str(self.dtype): 1})
        result = self.dtypes.value_counts()
        result.index = result.index.map(lambda x: str(x))
        return result
127235
def dtype_to_matlab_class(dtype):
    
    if dtype.str in ['<f8']:
        return 'double'
    elif dtype.str in ['<f4']:
        return 'single'
    elif dtype.str in ['<i8']:
        return 'int64'
    elif dtype.str in ['<u8']:
        return 'uint64'
    elif dtype.str in ['<i4']:
        return 'int32'
    elif dtype.str in ['<u4']:
        return 'uint32'
    elif dtype.str in ['<i2']:
        return 'int16'
    elif dtype.str in ['<u2']:
        return 'uint16'
    elif dtype.str in ['|i1']:
        return 'int8'
    elif dtype.str in ['|u1']:
        return 'uint8'
148417
def astype(self, col_dtypes, **kwargs):
        
        # Group indices to update by dtype for less map operations
        dtype_indices = {}
        columns = col_dtypes.keys()
        numeric_indices = list(self.columns.get_indexer_for(columns))
        # Create Series for the updated dtypes
        new_dtypes = self.dtypes.copy()
        for i, column in enumerate(columns):
            dtype = col_dtypes[column]
            if (
                not isinstance(dtype, type(self.dtypes[column]))
                or dtype != self.dtypes[column]
            ):
                # Only add dtype only if different
                if dtype in dtype_indices.keys():
                    dtype_indices[dtype].append(numeric_indices[i])
                else:
                    dtype_indices[dtype] = [numeric_indices[i]]
                # Update the new dtype series to the proper pandas dtype
                try:
                    new_dtype = np.dtype(dtype)
                except TypeError:
                    new_dtype = dtype
                if dtype != np.int32 and new_dtype == np.int32:
                    new_dtype = np.dtype("int64")
                elif dtype != np.float32 and new_dtype == np.float32:
                    new_dtype = np.dtype("float64")
                new_dtypes[column] = new_dtype
        # Update partitions for each dtype that is updated
        new_data = self.data
        for dtype in dtype_indices.keys():

            def astype(df, internal_indices=[]):
                block_dtypes = {}
                for ind in internal_indices:
                    block_dtypes[df.columns[ind]] = dtype
                return df.astype(block_dtypes)

            new_data = new_data.apply_func_to_select_indices(
                0, astype, dtype_indices[dtype], keep_remaining=True
            )
        return self.__constructor__(new_data, self.index, self.columns, new_dtypes)
33681
def _choose_float_dtype(dtype, has_offset):
    """"""
    # Keep float32 as-is.  Upcast half-precision to single-precision,
    # because float16 is "intended for storage but not computation"
    if dtype.itemsize <= 4 and np.issubdtype(dtype, np.floating):
        return np.float32
    # float32 can exactly represent all integers up to 24 bits
    if dtype.itemsize <= 2 and np.issubdtype(dtype, np.integer):
        # A scale factor is entirely safe (vanishing into the mantissa),
        # but a large integer offset could lead to loss of precision.
        # Sensitivity analysis can be tricky, so we just use a float64
        # if there's any offset at all - better unoptimised than wrong!
        if not has_offset:
            return np.float32
    # For all other types and circumstances, we just use float64.
    # (safe because eg. complex numbers are not supported in NetCDF)
    return np.float64
919307
def convert(dbus_obj):
    
    _isinstance = partial(isinstance, dbus_obj)
    ConvertType = namedtuple('ConvertType', 'pytype dbustypes')

    pyint = ConvertType(int, (dbus.Byte, dbus.Int16, dbus.Int32, dbus.Int64,
                              dbus.UInt16, dbus.UInt32, dbus.UInt64))
    pybool = ConvertType(bool, (dbus.Boolean, ))
    pyfloat = ConvertType(float, (dbus.Double, ))
    pylist = ConvertType(lambda _obj: list(map(convert, dbus_obj)),
                         (dbus.Array, ))
    pytuple = ConvertType(lambda _obj: tuple(map(convert, dbus_obj)),
                          (dbus.Struct, ))
    types_str = (dbus.ObjectPath, dbus.Signature, dbus.String)
    if not PY3:
        types_str += (dbus.UTF8String,)
    pystr = ConvertType(str if PY3 else unicode, types_str)

    pydict = ConvertType(
        lambda _obj: dict(zip(map(convert, dbus_obj.keys()),
                              map(convert, dbus_obj.values())
                              )
                          ),
        (dbus.Dictionary, )
    )

    for conv in (pyint, pybool, pyfloat, pylist, pytuple, pystr, pydict):
        if any(map(_isinstance, conv.dbustypes)):
            return conv.pytype(dbus_obj)
    else:
        return dbus_obj
1176667
def if_pandas_df_convert_to_numpy(obj):
    
    if pd is not None and isinstance(obj, pd.DataFrame):
        return obj.values
    else:
        return obj
530077
def _maybe_from_pandas(data, feature_names, feature_types):
    
    try:
        import pandas as pd
    except ImportError:
        return data, feature_names, feature_types

    if not isinstance(data, pd.DataFrame):
        return data, feature_names, feature_types

    dtypes = data.dtypes
    if not all(dtype.name in ('int64', 'float64', 'bool') for dtype in dtypes):
        raise ValueError('DataFrame.dtypes must be int, float or bool')

    if feature_names is None:
        feature_names = data.columns.format()
    if feature_types is None:
        mapper = {'int64': 'int', 'float64': 'q', 'bool': 'i'}
        feature_types = [mapper[dtype.name] for dtype in dtypes]
    data = data.values.astype('float')
    return data, feature_names, feature_types
387745
def _numpy_to_python(obj):
    
    if isinstance(obj, dict):
        return {k: _numpy_to_python(v) for k, v in obj.items()}
    elif isinstance(obj, (list, tuple, np.ndarray)):
        return [_numpy_to_python(x) for x in obj]
    elif isinstance(obj, FormattedFeatureName):
        return obj.value
    elif isinstance(obj, _numpy_string_types):
        return six.text_type(obj)
    elif hasattr(obj, 'dtype') and np.isscalar(obj):
        if np.issubdtype(obj, np.floating):
            return float(obj)
        elif np.issubdtype(obj, np.integer):
            return int(obj)
        elif np.issubdtype(obj, np.bool_):
            return bool(obj)
    return obj
429843
def _copy_with_dtype(data, dtype):
    
    result = np.empty(data.shape, dtype)
    result[...] = data
    return result
1020359
def to_pandas(self):
        
        df = self.data.to_pandas(is_transposed=self._is_transposed)
        if df.empty:
            dtype_dict = {
                col_name: pandas.Series(dtype=self.dtypes[col_name])
                for col_name in self.columns
            }
            df = pandas.DataFrame(dtype_dict, self.index)
        else:
            ErrorMessage.catch_bugs_and_request_email(
                len(df.index) != len(self.index) or len(df.columns) != len(self.columns)
            )
            df.index = self.index
            df.columns = self.columns
        return df
682199
def _to_numeric(self, offset=None, datetime_unit=None, dtype=float):
        
        numeric_array = duck_array_ops.datetime_to_numeric(
            self.data, offset, datetime_unit, dtype)
        return type(self)(self.dims, numeric_array, self._attrs)