Get train history as pd.DataFrame
552779
def train_history(self, tid=None):
        

        def result2history(result):
            if isinstance(result["history"], list):
                return pd.concat([pd.DataFrame(hist["loss"]).assign(fold=i)
                                  for i, hist in enumerate(result["history"])])
            else:
                return pd.DataFrame(result["history"]["loss"])

        # use all
        if tid is None:
            tid = self.valid_tid()

        res = [result2history(t["result"]).assign(tid=t["tid"]) for t in self.trials
               if t["tid"] in _listify(tid)]
        df = pd.concat(res)

        # reorder columns
        fold_name = ["fold"] if "fold" in df else []
        df = _put_first(df, ["tid"] + fold_name + ["epoch"])
        return df
54521
def retrieve_data(self):
        

        #==== Retrieve data ====#

        df = self.manager.get_historic_data(self.start.date(), self.end.date())
        df.replace(0, np.nan, inplace=True)
        return df
848602
def ts_to_df(metadata):
    
    logger_dataframes.info("enter ts_to_df")
    dfs = {}

    # Plot the variable + values vs year, age, depth (whichever are available)
    dfs["paleoData"] = pd.DataFrame(_plot_ts_cols(metadata))

    # Plot the chronology variables + values in a data frame
    dfs["chronData"] = _get_key_data(metadata, "chronData_df")

    # Take out the chronData pandas data frame object if it exists in the metadata
    # Otherwise, the data frame renderer gets crazy and errors out.
    if "chronData_df" in metadata:
        del metadata["chronData_df"]
    s = collections.OrderedDict(sorted(metadata.items()))

    # Put key-vars in a data frame to make it easier to visualize
    dfs["metadata"] = pd.DataFrame(list(s.items()), columns=['Key', 'Value'])

    logger_dataframes.info("exit ts_to_df")
    return dfs
960187
def get_eod_frame(self):
        """"""
        close = self.pxs.close
        mktval = self.get_mkt_val(close)
        dvds = self.pxs.dvds
        df = pd.DataFrame({'close': close, 'mkt_val': mktval, 'dvds': dvds})
        df.index.name = 'date'
        return df
1161423
def _to_file_format(self, iamc_index):
        """"""
        df = self.timeseries(iamc_index=iamc_index).reset_index()
        df = df.rename(columns={c: str(c).title() for c in df.columns})
        return df
421535
def to_pandas(self):
        
        dataframe = self.get().to_pandas()
        assert type(dataframe) is pandas.DataFrame or type(dataframe) is pandas.Series

        return dataframe
299184
def get_dataframes(symbols=("sne", "goog", "tsla"), source='yahoo', refresh=False):
    
    symbols = util.make_symbols(list(symbols))
    if refresh:
        symbols_to_refresh = symbols
    else:
        symbols_to_refresh = [sym for sym in symbols if not Equity.objects.filter(symbol=sym).exists()]
    source = source.lower().strip()
    if source in ('yahoo', 'google'):
        source += '_finance'
    if source[:3] == 'fed':
        source = 'federal_reserve_economic_data'
    ccpanda = ccp.ConcurrentPandas()
    # set the data source
    getattr(ccpanda, "set_source_" + source)()
    if symbols_to_refresh:
        # tell concurrent pandas which keys/symbols to retrieve
        ccpanda.insert_keys(symbols_to_refresh)
        # start concurrentpandas threads
        ccpanda.consume_keys_asynchronous_threads()
        # FIXME: is there a better/faster iterator to use like `ccpanda.output_map` attribute?
        pseudodict = ccpanda.return_map()
    else:
        pseudodict = {}
    table = {}
    for sym in symbols:
        e, created = None, False
        if not sym in symbols_to_refresh:
            e, created = Equity.objects.get_or_create(symbol=sym)
        if created or not e or not e.time_series or sym in symbols_to_refresh:
            e, created = Equity.objects.get_or_create(
                symbol=sym,
                name=sym,  # FIXME: use data source to find equity name!
                time_series=pseudodict[sym].to_json(),
                )
        table[sym] = pd.io.json.read_json(path_or_buf=e.time_series, orient='columns', typ='frame', convert_dates=True)
    return table
664445
def history(self):
        
        history = OrderedDict()
        for cycle in self:
            d = cycle.last_iteration
            for k, v in d.items():
                if k in history:
                    history[k].append(v)
                else:
                    history[k] = [v]

        # Convert to numpy arrays.
        for k, v in history.items():
            history[k] = np.array(v)

        return history
402495
def get_timeseries(self, *args, **kwargs):
        
        import predix.data.timeseries
        ts = predix.data.timeseries.TimeSeries(*args, **kwargs)
        return ts
307270
def get_data_with_timestamps(self):
        
        result = []
        for t, d in zip(self.timestamps, self.data_points):
            result.append(t, round(d, self.lr))
        return result
1034692
def to_dataframe(self):
        """"""
        def row_from_variant(variant):
            return OrderedDict([
                ("chr", variant.contig),
                ("start", variant.original_start),
                ("ref", variant.original_ref),
                ("alt", variant.original_alt),
                ("gene_name", ";".join(variant.gene_names)),
                ("gene_id", ";".join(variant.gene_ids))
            ])
        rows = [row_from_variant(v) for v in self]
        if len(rows) == 0:
            # TODO: return a DataFrame with the appropriate columns
            return pd.DataFrame()
        return pd.DataFrame.from_records(rows, columns=rows[0].keys())
1038644
def _fetch_dataframe(self):
        
        def reshape(training_summary):
            # Helper method to reshape a single training job summary into a dataframe record
            out = {}
            for k, v in training_summary['TunedHyperParameters'].items():
                # Something (bokeh?) gets confused with ints so convert to float
                try:
                    v = float(v)
                except (TypeError, ValueError):
                    pass
                out[k] = v
            out['TrainingJobName'] = training_summary['TrainingJobName']
            out['TrainingJobStatus'] = training_summary['TrainingJobStatus']
            out['FinalObjectiveValue'] = training_summary.get('FinalHyperParameterTuningJobObjectiveMetric',
                                                              {}).get('Value')

            start_time = training_summary.get('TrainingStartTime', None)
            end_time = training_summary.get('TrainingEndTime', None)
            out['TrainingStartTime'] = start_time
            out['TrainingEndTime'] = end_time
            if start_time and end_time:
                out['TrainingElapsedTimeSeconds'] = (end_time - start_time).total_seconds()
            return out
        # Run that helper over all the summaries.
        df = pd.DataFrame([reshape(tjs) for tjs in self.training_job_summaries()])
        return df
67052
def history(self, dates=None, linreg_since=None, lin_reg_days=20):
        '''
        
        '''
        dates = dates or self.get_dates_range()
        vals = [self.on_date(dt, only_count=True) for dt in dates]
        ret = Series(vals, index=dates)
        if linreg_since is not None:
            ret = self._linreg_future(ret, linreg_since, lin_reg_days)
        return ret.sort_index()
1311542
def _get_kdjk(cls, df, n_days):
        
        rsv_column = 'rsv_{}'.format(n_days)
        k_column = 'kdjk_{}'.format(n_days)
        df[k_column] = list(cls._calc_kd(df.get(rsv_column)))
942348
def tcounts(self):
        
        df = pd.DataFrame([[t.name(), t.size()] for t in self.tables()], columns=["name", "size"])
        df.index = df.name
        return df
1027780
def get_current(self):
        

        where = slice(self._oldest_frame_idx(), self._pos)
        major_axis = pd.DatetimeIndex(deepcopy(self.date_buf[where]), tz='utc')
        return pd.Panel(self.buffer.values[:, where, :], self.items,
                        major_axis, self.minor_axis, dtype=self.dtype)
471335
def dataframe(self):
        
        frames = []
        for game in self.__iter__():
            df = game.dataframe
            if df is not None:
                frames.append(df)
        if frames == []:
            return None
        return pd.concat(frames)