Return an RDD of Panda DataFrame objects. This can be expensive
        especially if we don't do a narrow transformation after and get it back
        to Spark SQL land quickly.
193921
def _rdd(self):
        """"""
        columns = self._schema_rdd.columns
        index_names = self._index_names

        def fromRecords(records):
            if not records:
                return []
            else:
                loaded_df = pd.DataFrame.from_records([records],
                                                      columns=columns)
                indexed_df = _update_index_on_df(loaded_df, index_names)
                return [indexed_df]

        return self._schema_rdd.rdd.flatMap(fromRecords)
1743
def fromDataFrameRDD(cls, rdd, sql_ctx):
        """"""
        result = DataFrame(None, sql_ctx)
        return result.from_rdd_of_dataframes(rdd)
1038605
def to_sql(cls, qc, **kwargs):
        
        # we first insert an empty DF in order to create the full table in the database
        # This also helps to validate the input against pandas
        # we would like to_sql() to complete only when all rows have been inserted into the database
        # since the mapping operation is non-blocking, each partition will return an empty DF
        # so at the end, the blocking operation will be this empty DF to_pandas

        empty_df = qc.head(1).to_pandas().head(0)
        empty_df.to_sql(**kwargs)
        # so each partition will append its respective DF
        kwargs["if_exists"] = "append"
        columns = qc.columns

        def func(df, **kwargs):
            df.columns = columns
            df.to_sql(**kwargs)
            return pandas.DataFrame()

        map_func = qc._prepare_method(func, **kwargs)
        result = qc._map_across_full_axis(1, map_func)
        # blocking operation
        result.to_pandas()
604647
def collect(self):
        """"""
        local_df = self._schema_rdd.toPandas()
        correct_idx_df = _update_index_on_df(local_df, self._index_names)
        return correct_idx_df
998112
def _return_samples(return_type, samples):
    
    if return_type.lower() == "dataframe":
        if HAS_PANDAS:
            return pandas.DataFrame.from_records(samples)
        else:
            warn("Pandas installation not found. Returning numpy.recarray object")
            return samples
    else:
        return samples
1161798
def _make_df(rows):
    """"""

    # Make DataFrame
    df = pd.DataFrame(rows).set_index('ts')

    # TimeDelta Support: https://issues.apache.org/jira/browse/ARROW-835
    for column in df.columns:
        if(df[column].dtype == 'timedelta64[ns]'):
            print('Converting timedelta column {:s}...'.format(column))
            df[column] = df[column].astype(str)
    return df
281916
def to_dataframe(self):
        
        keys = self.data[0].keys()
        column_list =[]
        for k in keys:
            key_list = []
            for i in xrange(0,len(self.data)):
                key_list.append(self.data[i][k])
            column_list.append(key_list)
        df = DataFrame(np.asarray(column_list).transpose(), columns=keys)
        for i in xrange(0,df.shape[1]):
            if is_number(df.iloc[:,i]):
                df.iloc[:,i] = df.iloc[:,i].astype(float)
        return df
316014
def convert_types_slow(df):
    ''''''
    dtypes = get_types(df)
    for k, v in dtypes.items():
        t = df[df['key']==k]
        t['value'] = t['value'].astype(v)
    df = df.apply(convert_row, axis=1)
    return df
711624
def build_conversion_table(self, dataframes):
        
        self.data = pd.DataFrame(dataframes)
        tmp_pairs = [s.split("/") for s in self.data.columns]
        self.data.columns = pd.MultiIndex.from_tuples(tmp_pairs)
1038590
def to_df(self)->None:
        ""
        return pd.DataFrame(dict(x=self.x._relative_item_paths(), y=[str(o) for o in self.y]))
1028540
def to_dataframe(self) -> "pandas.DataFrame":
        
        import pandas as pd
        df = pd.DataFrame(
            {
                "left": self.bin_left_edges,
                "right": self.bin_right_edges,
                "frequency": self.frequencies,
                "error": self.errors,
            },
            columns=["left", "right", "frequency", "error"])
        return df
838971
def toseries(self):
        
        from thunder.series.series import Series

        if self.mode == 'spark':
            values = self.values.values_to_keys(tuple(range(1, len(self.shape)))).unchunk()

        if self.mode == 'local':
            values = self.values.unchunk()
            values = rollaxis(values, 0, values.ndim)

        return Series(values)
149089
def _get_dataframe(self):
        
        if self.source_conn is None:  # Use local file
            return ge.read_csv(self.dataset_name, **self.dataset_params)

        if isinstance(self.source_conn, S3Hook):
            hook = ExpectationS3CsvHook(aws_conn_id=self.source_conn_id)

            return hook.get_ge_df(self.dataset_name, self.source_bucket_name, **self.dataset_params)

        if isinstance(self.source_conn, DbApiHook):
            hook = ExpectationMySQLHook(mysql_conn_id=self.source_conn_id)

            return hook.get_ge_df(self.dataset_name, **self.dataset_params)
1034578
def to_pandas(self):
        
        df = self.data.to_pandas(is_transposed=self._is_transposed)
        if df.empty:
            if len(self.columns) != 0:
                df = pandas.DataFrame(columns=self.columns).astype(self.dtypes)
            else:
                df = pandas.DataFrame(columns=self.columns, index=self.index)
        else:
            ErrorMessage.catch_bugs_and_request_email(
                len(df.index) != len(self.index) or len(df.columns) != len(self.columns)
            )
            df.index = self.index
            df.columns = self.columns
        return df
456519
def _py2java(sc, obj):
    
    if isinstance(obj, RDD):
        obj = _to_java_object_rdd(obj)
    elif isinstance(obj, DataFrame):
        obj = obj._jdf
    elif isinstance(obj, SparkContext):
        obj = obj._jsc
    elif isinstance(obj, list):
        obj = [_py2java(sc, x) for x in obj]
    elif isinstance(obj, JavaObject):
        pass
    elif isinstance(obj, (int, long, float, bool, bytes, unicode)):
        pass
    else:
        data = bytearray(PickleSerializer().dumps(obj))
        obj = sc._jvm.org.apache.spark.mllib.api.python.SerDe.loads(data)
    return obj
1010571
def make_dataframe(obj, columns=None, exclude=None, limit=1e8):
    """"""
    try:
        obj = obj.objects.all()[:limit]
    except:
        pass
    if isinstance(obj, (pd.Series, list, tuple)):
        return make_dataframe(pd.DataFrame(obj), columns, exclude, limit)
    # if the obj is a named tuple, DataFrame, dict of columns, django QuerySet, sql alchemy query result
    # retrieve the "include"d field/column names from its keys/fields/attributes
    if columns is None:
        columns = get_column_labels(obj)
    if exclude is not None and columns is not None and columns and exclude:
        columns = [i for i in columns if i not in exclude]
    try:
        return pd.DataFrame(list(obj.values(*columns)[:limit]))
    except:
        pass
    try:
        return pd.DataFrame(obj)[fuzzy_get(obj, columns)]
    except:
        pass
    return pd.DataFrame(obj)
409256
def to_pandas_dataframe(self):
        
        pd_index = self.index().to_pandas_index()
        return pd.DataFrame.from_items(self.collect()).set_index(pd_index)