Plot Panda dataframe.

  :param table: Panda dataframe returned by :func:`analyzeWeightPruning`
  :type table: :class:`pandas.DataFrame`
  :param title: Plot title
  :type title: str
  :param plotPath: Plot full path
  :type plotPath: str
423032
def plotDataframe(table, title, plotPath):
  
  plt.figure()
  axes = table.T.plot(subplots=True, sharex=True, grid=True, legend=True,
                      title=title, figsize=(8, 11))

  # Use fixed scale for "accuracy"
  accuracy = next(ax for ax in axes if ax.lines[0].get_label() == 'accuracy')
  accuracy.set_ylim(0.0, 1.0)

  plt.savefig(plotPath)
  plt.close()
1115992
def dataframe_from_result_table(table):
    import pandas as pd
    from ._models import KustoResultTable
    from dateutil.tz import UTC

    
    if not table:
        raise ValueError()

    if not isinstance(table, KustoResultTable):
        raise TypeError("Expected KustoResultTable got {}".format(type(table).__name__))

    columns = [col.column_name for col in table.columns]
    frame = pd.DataFrame(table._rows, columns=columns)

    # fix types
    for col in table.columns:
        if col.column_type == "bool":
            frame[col.column_name] = frame[col.column_name].astype(bool)

    return frame
274485
def apply(self, df):
        """"""

        if self.subset:
            if _axis_is_rows(self.axis):
                df = df[self.subset]
            if _axis_is_cols(self.axis):
                df = df.loc[self.subset]

        result = df.agg(self.func, axis=self.axis, *self.args, **self.kwargs)
        result.name = self.title
        return result
1049960
def table_as_df(self, table_name):
        """"""
        self.table_must_exist(table_name)
        query = "SELECT * FROM `%s`" % table_name.lower()
        return pandas.read_sql(query, self.own_conn)
1220552
def schema(self, table):
        
        try:
            pprint(self.query("PRAGMA table_info({})".format(table), fmt='table'))
        except ValueError:
            print('Table {} not found'.format(table))
368023
def set_title(self, row, column, text):
        
        subplot = self.get_subplot_at(row, column)
        subplot.set_title(text)
1269506
def load_data(self, table_name, obj, **kwargs):
        
        # kwargs is a catch all for any options required by other backends.
        self.dictionary[table_name] = pd.DataFrame(obj)
495559
def _open_dataset(self, urlpath):
        
        import dask.dataframe

        if self.pattern is None:
            self._dataframe = dask.dataframe.read_csv(
                urlpath, storage_options=self._storage_options,
                **self._csv_kwargs)
            return

        if not (DASK_VERSION >= '0.19.0'):
            raise ValueError("Your version of dask is '{}'. "
                "The ability to include filenames in read_csv output "
                "(``include_path_column``) was added in 0.19.0, so "
                "pattern urlpaths are not supported.".format(DASK_VERSION))

        drop_path_column = 'include_path_column' not in self._csv_kwargs
        path_column = self._path_column()

        self._dataframe = dask.dataframe.read_csv(
            urlpath, storage_options=self._storage_options, **self._csv_kwargs)

        # add the new columns to the dataframe
        self._set_pattern_columns(path_column)

        if drop_path_column:
            self._dataframe = self._dataframe.drop([path_column], axis=1)
1166765
def read(self, dataframe_name: str) -> pandas.DataFrame:
        
        code = serialise_dataframe_code(dataframe_name, self.kind)
        output = self._execute(code)
        output.raise_for_status()
        if output.text is None:
            raise RuntimeError("statement had no text output")
        return deserialise_dataframe(output.text)
1038644
def _fetch_dataframe(self):
        
        def reshape(training_summary):
            # Helper method to reshape a single training job summary into a dataframe record
            out = {}
            for k, v in training_summary['TunedHyperParameters'].items():
                # Something (bokeh?) gets confused with ints so convert to float
                try:
                    v = float(v)
                except (TypeError, ValueError):
                    pass
                out[k] = v
            out['TrainingJobName'] = training_summary['TrainingJobName']
            out['TrainingJobStatus'] = training_summary['TrainingJobStatus']
            out['FinalObjectiveValue'] = training_summary.get('FinalHyperParameterTuningJobObjectiveMetric',
                                                              {}).get('Value')

            start_time = training_summary.get('TrainingStartTime', None)
            end_time = training_summary.get('TrainingEndTime', None)
            out['TrainingStartTime'] = start_time
            out['TrainingEndTime'] = end_time
            if start_time and end_time:
                out['TrainingElapsedTimeSeconds'] = (end_time - start_time).total_seconds()
            return out
        # Run that helper over all the summaries.
        df = pd.DataFrame([reshape(tjs) for tjs in self.training_job_summaries()])
        return df
162921
def line_plot(self, x='year', y='value', **kwargs):
        
        df = self.as_pandas(with_metadata=kwargs)

        # pivot data if asked for explicit variable name
        variables = df['variable'].unique()
        if x in variables or y in variables:
            keep_vars = set([x, y]) & set(variables)
            df = df[df['variable'].isin(keep_vars)]
            idx = list(set(df.columns) - set(['value']))
            df = (df
                  .reset_index()
                  .set_index(idx)
                  .value  # df -> series
                  .unstack(level='variable')  # keep_vars are columns
                  .rename_axis(None, axis=1)  # rm column index name
                  .reset_index()
                  .set_index(META_IDX)
                  )
            if x != 'year' and y != 'year':
                df = df.drop('year', axis=1)  # years causes NaNs

        ax, handles, labels = plotting.line_plot(
            df.dropna(), x=x, y=y, **kwargs)
        return ax
464318
def plot(self, df_data, center=False, save=False,
             save_name=None, save_path='saved', dated=True, notebook=True):
        ""
        data = df_data.to_json(orient='records')
        options = self.to_dict()
        return plot(data, options, center=center, save=save,
                    save_name=save_name, save_path=save_path, dated=dated, notebook=notebook)
344887
def get_dataframe():
        
        names = []
        [names.extend(line.split()) for line in CONTROL_VARIABLE_LINES]

        defaults = []
        [defaults.extend(line.split()) for line in CONTROL_DEFAULT_LINES]

        types, required,cast_defaults,formats = [],[],[],[]
        for name,default in zip(names,defaults):
            if '[' in name or ']' in name:
                required.append(False)
            else:
                required.append(True)
            v,t,f = ControlData._parse_value(default)
            types.append(t)
            formats.append(f)
            cast_defaults.append(v)
        return pandas.DataFrame({"name":names,"type":types,
                                     "value":cast_defaults,"required":required,
                                    "format":formats})
1304367
def insert_table(self, label = None, name = None, **kwargs):
        

        data_frame = kwargs.pop('data_frame', None)
        if data_frame is None:
            data_frame = kwargs.pop('dataframe', None)

        to_hdf_kwargs = kwargs.pop('to_hdf_kwargs', dict())
        if data_frame is not None:
            assert isinstance(data_frame, pandas.DataFrame)

        if data_frame is not None:
            if label is None:
                label = name
            table = Table(label = label, name = name, survey = self)
            assert table.survey.hdf5_file_path is not None
            log.debug("Saving table {} in {}".format(name, table.survey.hdf5_file_path))
            table.save_data_frame(data_frame, **to_hdf_kwargs)

        if name not in self.tables:
            self.tables[name] = dict()
        for key, val in kwargs.items():
            self.tables[name][key] = val
781606
def dataframe(self, dtype=False, parse_dates=True,  *args, **kwargs):
        """"""

        import pandas as pd

        rg = self.row_generator

        t = self.resolved_url.get_resource().get_target()

        if t.target_format == 'csv':
            return self.read_csv(dtype, parse_dates, *args, **kwargs)

        # Maybe generator has it's own Dataframe method()
        try:

            return rg.dataframe( *args, **kwargs)
        except AttributeError:
            pass

        # Just normal data, so use the iterator in this object.
        headers = next(islice(self, 0, 1))
        data = islice(self, 1, None)

        df = pd.DataFrame(list(data), columns=headers,  *args, **kwargs)

        self.errors = df.metatab_errors = rg.errors if hasattr(rg, 'errors') and rg.errors else {}

        return df
911865
def pp_tpl_to_dataframe(tpl_filename):
    
    inlines = open(tpl_filename, 'r').readlines()
    header = inlines.pop(0)
    marker = header.strip().split()[1]
    assert len(marker) == 1
    usecols = [0,1,2,3]
    df = pd.read_csv(tpl_filename, delim_whitespace=True,skiprows=1,
                     header=None, names=PP_NAMES[:-1],usecols=usecols)
    df.loc[:,"name"] = df.name.apply(str).apply(str.lower)
    df["parnme"] = [i.split(marker)[1].strip() for i in inlines]


    return df
1238640
def dtypes_summary(df):
    
    output_df = pd.DataFrame([])
    row_count = df.shape[0]
    row_indexes = ['rows_numerical','rows_string','rows_date_time','category_count','largest_category','rows_na','rows_total']
    for colname in df:
        data = df[colname] # data is the pandas series associated with this column
        # number of numerical values in the column
        rows_numerical = pd.to_numeric(data,errors = 'coerce').count()
        # number of values that can't be coerced to a numerical
        rows_string = row_count - rows_numerical
        # number of values that can be coerced to a date-time object
        rows_date_time = pd.to_datetime(data,errors = 'coerce',infer_datetime_format = True).count()
        # categories in column
        value_counts = data.value_counts().reset_index()
        # number of different values in the dataframe
        categories = len(value_counts)
        # largest category
        largest_category = value_counts.iloc[0,1]
        # number of null/missing values
        rows_na = data.isnull().sum()
        # build the output list
        output_data = [rows_numerical, rows_string, rows_date_time, categories, 
                       largest_category,rows_na,row_count]
        # add to dataframe
        output_df.loc[:,colname] = pd.Series(output_data)

    # row names
    output_df.index = row_indexes
    return output_df