acts like rbind for pandas dataframes
191297
def rbind(dfs):
    
    if len(dfs) == 1:
        return dfs[0]
    df = dfs[0]
    for d in dfs[1:]:
        df = df.append(d)
    return df
161360
def transform(self, df):
        """"""

        # Make sure the dataframe columns are the same as the ones passed to fit
        assert df.columns.equals(self.columns_)

        # Convert the dataframe with get_dummies
        df_with_dummies = pd.get_dummies(df)

        # Make sure our columns exactly match what we stored during the fit operation
        my_columns = set(df_with_dummies.columns.tolist())
        output_column_set = set(self.columns_in_order)
        for column in my_columns.difference(output_column_set):
            print('Deleting extra column {:s}'.format(column))
            del df_with_dummies[column]
        for column in output_column_set.difference(my_columns):
            print('Adding column {:s}'.format(column))
            df_with_dummies[column] = 0

        # Returns the same columns in the same order every time
        return np.asarray(df_with_dummies[self.columns_in_order])
688925
def _get_processed_dataframe(self, dataframe):
        
        dataframe.index = pd.to_datetime(dataframe['epoch'], unit='s', utc=True)
        del dataframe['epoch']
        summary = dataframe.describe(percentiles=[.80, .90, .95]).transpose().loc['scriptrun_time']
        df_grp = dataframe.groupby(pd.TimeGrouper('{}S'.format(self.interval)))
        df_final = df_grp.apply(lambda x: x.describe(percentiles=[.80, .90, .95])['scriptrun_time'])

        return {
            "raw": dataframe.round(2),
            "compiled": df_final.round(2),
            "summary": summary.round(2)
        }
1198859
def apply_filters(df, filters):
        """"""
        idx = pd.Series([True]*df.shape[0])
        for k, v in list(filters.items()):
            if k not in df.columns:
                continue
            idx &= (df[k] == v)

        return df.loc[idx]
990319
def is_dataframe(data):
    
    dd = None
    if 'dask' in sys.modules and 'pandas' in sys.modules:
        import dask.dataframe as dd
    return((pd is not None and isinstance(data, pd.DataFrame)) or
          (dd is not None and isinstance(data, dd.DataFrame)))
290527
def _maybe_numeric_slice(df, slice_, include_bool=False):
    
    if slice_ is None:
        dtypes = [np.number]
        if include_bool:
            dtypes.append(bool)
        slice_ = IndexSlice[:, df.select_dtypes(include=dtypes).columns]
    return slice_
264506
def avail(df):
    """"""
    avail = DataFrame(
        {
            "start": df.apply(lambda col: col.first_valid_index()),
            "end": df.apply(lambda col: col.last_valid_index()),
        }
    )
    return avail[["start", "end"]]
467694
def from_rdd_of_dataframes(self, rdd, column_idxs=None):
        """"""
        def frame_to_spark_sql(frame):
            """Convert a Panda's DataFrame into Spark SQL Rows"""
            return [r.tolist() for r in frame.to_records()]

        def frame_to_schema_and_idx_names(frames):
            """Returns the schema and index names of the frames. Useful
            if the frame is large and we wish to avoid transfering
            the entire frame. Only bothers to apply once per partiton"""
            try:
                frame = frames.next()
                return [(list(frame.columns), list(frame.index.names))]
            except StopIteration:
                return []

        # Store if the RDD was persisted so we don't uncache an
        # explicitly cached input.
        was_persisted = rdd.is_cached
        # If we haven't been supplied with the schema info cache the RDD
        # since we are going to eveluate the first partition and then eveluate
        # the entire RDD as part of creating a Spark DataFrame.
        (schema, index_names) = ([], [])
        if not column_idxs:
            rdd.cache()
            (schema, index_names) = rdd.mapPartitions(
                frame_to_schema_and_idx_names).first()
        else:
            (schema, index_names) = column_idxs
        # Add the index_names to the schema.
        index_names = _normalize_index_names(index_names)
        schema = index_names + schema
        ddf = DataFrame.from_schema_rdd(
            self.sql_ctx.createDataFrame(rdd.flatMap(frame_to_spark_sql),
                                         schema=schema))
        ddf._index_names = index_names
        if not was_persisted:
            rdd.unpersist()
        return ddf
804891
def _to_df(self, query, conn, index_col=None, coerce_float=True, params=None,
               parse_dates=None, columns=None):
        
        return pd.io.sql.read_sql(str(query), conn, index_col=index_col,
                                  coerce_float=coerce_float, params=params,
                                  parse_dates=parse_dates, columns=columns)
1021418
def roll_sd(input, window, scale = 1.0, ddof = 0):
    ''''''
    nobs, i, j, sx, sxx = 0,0,0,0.,0.
    N = len(input)
    sqrt = np.sqrt

    if window > N:
        raise ValueError('Out of bound')

    output = np.ndarray(N-window+1,dtype=input.dtype)

    for val in input[:window]:
        if val == val:
            nobs += 1
            sx += val
            sxx += val*val

    nn = nobs - ddof
    output[j] = NaN if nn<=0 else sqrt(scale * (sxx - sx*sx/nobs) / nn)

    for val in input[window:]:
        prev = input[j]
        if prev == prev:
            sx -= prev
            sxx -= prev*prev
            nobs -= 1

        if val == val:
            nobs += 1
            sx += val
            sxx += val*val

        j += 1
        nn = nobs - ddof
        output[j] = NaN if nn<=0 else sqrt(scale * (sxx - sx*sx/nobs) / nn)

    return output
154698
def _prep_pandas_groupby(self):
        """"""
        myargs = self._myargs
        mykwargs = self._mykwargs

        def extract_keys(groupedFrame):
            for key, group in groupedFrame:
                yield (key, group)

        def group_and_extract(frame):
            return extract_keys(frame.groupby(*myargs, **mykwargs))

        self._baseRDD = self._prdd._rdd()
        self._distributedRDD = self._baseRDD.flatMap(group_and_extract)
        self._mergedRDD = self._sortIfNeeded(
            self._group(self._distributedRDD))
1078972
def gen_rsd_cdf(K, delta, c):
    """"""

    mu = gen_mu(K, delta, c)
    return [sum(mu[:d+1]) for d in range(K)]
731742
def _wrap_result(data, columns, index_col=None, coerce_float=True,
                 parse_dates=None):
    """"""

    frame = DataFrame.from_records(data, columns=columns,
                                   coerce_float=coerce_float)

    frame = _parse_date_columns(frame, parse_dates)

    if index_col is not None:
        frame.set_index(index_col, inplace=True)

    return frame
983991
def flatten(self, df, column_name):
        """"""
        _exp_list = [[md5, x] for md5, value_list in zip(df['md5'], df[column_name]) for x in value_list]
        return pd.DataFrame(_exp_list, columns=['md5',column_name])
1743
def fromDataFrameRDD(cls, rdd, sql_ctx):
        """"""
        result = DataFrame(None, sql_ctx)
        return result.from_rdd_of_dataframes(rdd)
828861
def dataframe_from_list(values):
    """"""
    if(isinstance(values, six.string_types)):
        return DataFrame(filename=values)
    elif(isinstance(values, fits.HDUList)):
        return DataFrame(frame=values)
    else:
        return None
66169
def from_dataframe(cls, name, df, indices, primary_key=None):
        """"""

        # ordered list (column_name, column_type) pairs
        column_types = []
        # which columns have nullable values
        nullable = set()

        # tag cached database by dataframe's number of rows and columns
        for column_name in df.columns:
            values = df[column_name]
            if values.isnull().any():
                nullable.add(column_name)
            column_db_type = db_type(values.dtype)
            column_types.append((column_name.replace(" ", "_"), column_db_type))

        def make_rows():
            return list(tuple(row) for row in df.values)

        return cls(
            name=name,
            column_types=column_types,
            make_rows=make_rows,
            indices=indices,
            nullable=nullable,
            primary_key=primary_key)