Read in the Berger and Loutre orbital table as a pandas dataframe, convert to xarray
332184
def _get_Berger_data(verbose=True):
    '''
    '''
    # The first column of the data file is used as the row index, and represents kyr from present
    orbit91_pd, path = load_data_source(local_path = local_path,
                    remote_source_list = [threddspath, NCDCpath],
                    open_method = pd.read_csv,
                    open_method_kwargs = {'delim_whitespace': True, 'skiprows':1},
                    verbose=verbose,)
    #  As xarray structure with the dimension named 'kyear'
    orbit = xr.Dataset(orbit91_pd).rename({'dim_0': 'kyear'})
    #  Now change names
    orbit = orbit.rename({'ECC': 'ecc', 'OMEGA': 'long_peri',
                          'OBL': 'obliquity', 'PREC': 'precession'})
    # add 180 degrees to long_peri (see lambda definition, Berger 1978 Appendix)
    orbit['long_peri'] += 180.
    orbit['precession'] *= -1.

    orbit.attrs['Description'] = 'The Berger and Loutre (1991) orbital data table'
    orbit.attrs['Citation'] = 'https://doi.org/10.1016/0277-3791(91)90033-Q'
    orbit.attrs['Source'] = path
    orbit.attrs['Note'] = 'Longitude of perihelion is defined to be 0 degrees at Northern Vernal Equinox. This differs by 180 degrees from orbit91 source file.'
    return orbit
871743
def posterior_to_xarray(self):
        """"""
        var_names = self.pymc3.util.get_default_varnames(  # pylint: disable=no-member
            self.trace.varnames, include_transformed=False
        )
        data = {}
        for var_name in var_names:
            data[var_name] = np.array(self.trace.get_values(var_name, combine=False, squeeze=False))
        return dict_to_dataset(data, library=self.pymc3, coords=self.coords, dims=self.dims)
574716
def _shape(self, df):
        

        row, col = df.shape
        return row + df.columns.nlevels, col + df.index.nlevels
1252459
def sample_stats_to_xarray(self):
        """"""
        posterior = self.posterior
        posterior_model = self.posterior_model
        # copy dims and coords
        dims = deepcopy(self.dims) if self.dims is not None else {}
        coords = deepcopy(self.coords) if self.coords is not None else {}

        # log_likelihood
        log_likelihood = self.log_likelihood
        if log_likelihood is not None:
            if isinstance(log_likelihood, str) and log_likelihood in dims:
                dims["log_likelihood"] = dims.pop(log_likelihood)

        data = get_sample_stats_stan3(
            posterior, model=posterior_model, log_likelihood=log_likelihood
        )

        return dict_to_dataset(data, library=self.stan, coords=coords, dims=dims)
42414
def row_coordinates(self, X):
        """"""
        utils.validation.check_is_fitted(self, 'V_')

        _, row_names, _, _ = util.make_labels_and_names(X)

        if isinstance(X, pd.SparseDataFrame):
            X = X.to_coo().astype(float)
        elif isinstance(X, pd.DataFrame):
            X = X.to_numpy()

        if self.copy:
            X = X.copy()

        # Normalise the rows so that they sum up to 1
        if isinstance(X, np.ndarray):
            X = X / X.sum(axis=1)[:, None]
        else:
            X = X / X.sum(axis=1)

        return pd.DataFrame(
            data=X @ sparse.diags(self.col_masses_.to_numpy() ** -0.5) @ self.V_.T,
            index=row_names
        )
804891
def _to_df(self, query, conn, index_col=None, coerce_float=True, params=None,
               parse_dates=None, columns=None):
        
        return pd.io.sql.read_sql(str(query), conn, index_col=index_col,
                                  coerce_float=coerce_float, params=params,
                                  parse_dates=parse_dates, columns=columns)
1015722
def _read_xlsx_table(path):
    """"""
    workbook = pyxl.load_workbook(path)
    worksheet = workbook.active
    table = helpers.sheet_to_table(worksheet)

    return table
922870
def parquet_to_df(filename, use_threads=1):
    
    try:
        return pq.read_table(filename, use_threads=use_threads).to_pandas()
    except pa.lib.ArrowIOError:
        print('Could not read parquet file {:s}'.format(filename))
        return None
669522
def posterior_to_xarray(self):
        """"""
        posterior = self.posterior
        # filter posterior_predictive and log_likelihood
        posterior_predictive = self.posterior_predictive
        if posterior_predictive is None:
            posterior_predictive = []
        elif isinstance(posterior_predictive, str):
            posterior_predictive = [posterior_predictive]
        log_likelihood = self.log_likelihood
        if not isinstance(log_likelihood, str):
            log_likelihood = []
        else:
            log_likelihood = [log_likelihood]

        ignore = posterior_predictive + log_likelihood + ["lp__"]

        data = get_draws(posterior, ignore=ignore)

        return dict_to_dataset(data, library=self.pystan, coords=self.coords, dims=self.dims)
1200943
def toArray(self):
        
        A = np.zeros((self.numRows, self.numCols), dtype=np.float64, order='F')
        for k in xrange(self.colPtrs.size - 1):
            startptr = self.colPtrs[k]
            endptr = self.colPtrs[k + 1]
            if self.isTransposed:
                A[k, self.rowIndices[startptr:endptr]] = self.values[startptr:endptr]
            else:
                A[self.rowIndices[startptr:endptr], k] = self.values[startptr:endptr]
        return A
1141463
def cache_to_df(dir_path):
    
    table = {}
    for attrib in glob.glob(os.path.join(dir_path, '*')):
        attrib_name, attrib_ext = os.path.splitext(os.path.basename(attrib))
        if attrib_ext == '.lf8':
            attrib_data = np.fromfile(attrib, np.float64)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.lf4':
            attrib_data = np.fromfile(attrib, np.float32)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.li2':
            attrib_data = np.fromfile(attrib, np.int16)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.li4':
            attrib_data = np.fromfile(attrib, np.int32)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.li8':
            attrib_data = np.fromfile(attrib, np.int64)
            table[attrib_name] = attrib_data

        elif attrib_ext == '.ib1':
            attrib_data = np.fromfile(attrib, np.bool_)
            table[attrib_name] = attrib_data

        elif attrib_ext.startswith('.iS'):
            length_string = int(attrib_ext[3:])
            attrib_data = np.fromfile(attrib, ('a' + str(length_string)))
            table[attrib_name] = attrib_data

        else:
            print('Array {} is not a recognized data type'.format(attrib))

    df = pd.DataFrame(table)
    return df
941894
def ascolumn(x, dtype = None):
    ''''''
    x = asarray(x, dtype)
    return x if len(x.shape) >= 2 else x.reshape(len(x),1)
1042169
def Field_to_xarray(field):
    ''''''
    dom = field.domain
    dims = []; dimlist = []; coords = {};
    for axname in dom.axes:
        dimlist.append(axname)
        try:
            assert field.interfaces[dom.axis_index[axname]]
            bounds_name = axname + '_bounds'
            dims.append(bounds_name)
            coords[bounds_name] = dom.axes[axname].bounds
        except:
            dims.append(axname)
            coords[axname] = dom.axes[axname].points
    #  Might need to reorder the data
    da = DataArray(field.transpose([dom.axis_index[name] for name in dimlist]),
                      dims=dims, coords=coords)
    for name in dims:
        try:
            da[name].attrs['units'] = dom.axes[name].units
        except:
            pass
    return da
640470
def posterior_predictive_to_xarray(self):
        """"""
        posterior_predictive = self.posterior_predictive
        columns = self.posterior[0].columns
        if (
            isinstance(posterior_predictive, (tuple, list))
            and posterior_predictive[0].endswith(".csv")
        ) or (isinstance(posterior_predictive, str) and posterior_predictive.endswith(".csv")):
            if isinstance(posterior_predictive, str):
                posterior_predictive = [posterior_predictive]
            chain_data = []
            for path in posterior_predictive:
                parsed_output = _read_output(path)
                for sample, *_ in parsed_output:
                    chain_data.append(sample)
            data = _unpack_dataframes(chain_data)
        else:
            if isinstance(posterior_predictive, str):
                posterior_predictive = [posterior_predictive]
            posterior_predictive_cols = [
                col
                for col in columns
                if any(item == col.split(".")[0] for item in posterior_predictive)
            ]
            data = _unpack_dataframes([item[posterior_predictive_cols] for item in self.posterior])
        return dict_to_dataset(data, coords=self.coords, dims=self.dims)
923241
def mbox_to_pandas(mbox_path):
    
    if os.path.isfile(mbox_path):
        mbox_files = [mbox_path]
    else:
        mbox_files = [os.path.join(dirpath, f) for dirpath, dirnames, files in os.walk(mbox_path) for f in files if f.endswith('mbox')]

    mail_table = []

    f_pbar = tqdm.tqdm(range(0,len(mbox_files)))
    f_pbar.set_description('Extracting mbox files...')

    for mbox_file in mbox_files:
        write_table(mbox_file, mail_table)
        f_pbar.update(1)

    df_out = pd.DataFrame(mail_table)
    df_out.columns = ['From', 'To', 'Cc', 'Date', 'Subject', 'Body']
    df_out['NumTo'] = df_out['To'].map(lambda i: len(i))
    df_out['NumCC'] = df_out['Cc'].map(lambda i: len(i))
    return df_out
199198
def index_to_loc(self, index):
        
        sz = int(self.shape[0] * self.shape[1])
        idxs = np.array([i for i in range(sz)], dtype=int).reshape(self.shape)
        args = np.argwhere(idxs == index)
        if len(args) < 1:
            raise RuntimeError('Index ({}) is out of range.')
        return args[0]
1298272
def point_coll_value(coll, xy, scale=1):
    """"""
    output = getinfo(coll.getRegion(ee.Geometry.Point(xy), scale=scale))

    # Structure output to easily be converted to a Pandas dataframe
    # First key is band name, second key is the date string
    col_dict = {}
    info_dict = {}
    for i, k in enumerate(output[0][4:]):
        col_dict[k] = i + 4
        info_dict[k] = {}
    for row in output[1:]:
        date = datetime.datetime.utcfromtimestamp(row[3] / 1000.0).strftime(
            '%Y-%m-%d')
        for k, v in col_dict.items():
            info_dict[k][date] = row[col_dict[k]]
    return info_dict