Loads the forecast files and gathers the forecast information into pandas DataFrames.
547777
def load_forecasts(self):
        
        forecast_path = self.forecast_json_path + "/{0}/{1}/".format(self.run_date.strftime("%Y%m%d"),
                                                                     self.ensemble_member)
        forecast_files = sorted(glob(forecast_path + "*.json"))
        for forecast_file in forecast_files:
            file_obj = open(forecast_file)
            json_obj = json.load(file_obj)
            file_obj.close()
            track_id = json_obj['properties']["id"]
            obs_track_id = json_obj['properties']["obs_track_id"]
            forecast_hours = json_obj['properties']['times']
            duration = json_obj['properties']['duration']
            for f, feature in enumerate(json_obj['features']):
                area = np.sum(feature["properties"]["masks"])
                step_id = track_id + "_{0:02d}".format(f)
                for model_type in self.model_types:
                    for model_name in self.model_names[model_type]:
                        prediction = feature['properties'][model_type + "_" + model_name.replace(" ", "-")]
                        if model_type == "condition":
                            prediction = [prediction]
                        row = [track_id, obs_track_id, self.ensemble_name, self.ensemble_member, forecast_hours[f],
                               f + 1, duration, area] + prediction
                        self.forecasts[model_type][model_name].loc[step_id] = row
1182268
def _load_files(self):
        """"""

        def copy_dir(path):
            for (dr, _, files) in walk(path):
                for fn in files:

                    if '__pycache__' in fn:
                        continue

                    relpath = dr.replace(self.source_dir, '').strip('/')
                    src = parse_app_url(join(dr, fn))
                    dest = join(relpath, fn)

                    resource = src.get_resource()

                    self._load_file( dest, resource.read())

        for term in self.resources(term = 'Root.Pythonlib'):

            uv = parse_app_url(term.value)
            ur = parse_app_url(self.source_dir)

            # In the case that the input doc is a file, and the ref is to a file,
            # try interpreting the file as relative.
            if ur.proto == 'file' and uv.proto == 'file':

                # Either a file or a directory
                path = join(self.source_dir, uv.path)
                if isdir(path):
                    copy_dir(path)

            else:
                # Load it as a URL
                f = self._get_ref_contents(term)
                try:
                    self._load_file(term.value,f.read() )
                except Exception as e:
                    raise PackageError("Failed to load file for '{}': {} ".format(term.value, e))

        nb_dir = join(self.source_dir, 'notebooks')

        if exists(nb_dir) and isdir(nb_dir):
            copy_dir(nb_dir)
706514
def run_forecast(self):

        
        # ----------------------------------------------------------------------
        # LSM to GSSHA
        # ----------------------------------------------------------------------
        self.prepare_hmet()
        self.prepare_gag()

        # ----------------------------------------------------------------------
        # RAPID to GSSHA
        # ----------------------------------------------------------------------
        self.rapid_to_gssha()

        # ----------------------------------------------------------------------
        # HOTSTART
        # ----------------------------------------------------------------------
        self.hotstart()

        # ----------------------------------------------------------------------
        # Run GSSHA
        # ----------------------------------------------------------------------
        return self.run()
636097
def load_df_from_file(file_path, sep=",", header=0):
  """"""
  with tf.gfile.Open(file_path) as infile:
    df = pd.read_csv(infile, sep=sep, header=header)
  return df
1117397
def _get_data_from_csv_files(self):
    """"""

    all_df = []
    for file_name in self._input_csv_files:
      with _util.open_local_or_gcs(file_name, mode='r') as f:
        all_df.append(pd.read_csv(f, names=self._headers))
    df = pd.concat(all_df, ignore_index=True)
    return df
162352
def create_preprocessed_dataset(path_to_data, outputpath, preprocessing_queue):
    """"""
    # Log everything
    logging.info("Data soure %s", path_to_data)
    logging.info("Output will be stored in %s", outputpath)
    tmp = "Preprocessing Queue:\n"
    for preprocessing_class in preprocessing_queue:
        tmp += str(preprocessing_class) + "\n"
    logging.info(tmp)
    # Load from pickled file
    if not os.path.isfile(path_to_data):
        logging.info(("'%s' does not exist. Please either abort this script "
                      "or update the data location."), path_to_data)
        raw_dataset_path = utils.choose_raw_dataset()
        # Get project-relative path
        raw_dataset_path = "raw-datasets" + \
                           raw_dataset_path.split("raw-datasets")[1]
        print(raw_dataset_path)
        sys.exit()  # TODO: Update model!
    logging.info("Start loading data...")
    loaded = pickle.load(open(path_to_data, "rb"))
    raw_datasets = loaded['handwriting_datasets']
    logging.info("Start applying preprocessing methods")
    start_time = time.time()
    for i, raw_dataset in enumerate(raw_datasets):
        if i % 10 == 0 and i > 0:
            utils.print_status(len(raw_datasets), i, start_time)
        # Do the work
        raw_dataset['handwriting'].preprocessing(preprocessing_queue)
    sys.stdout.write("\r%0.2f%% (done)\033[K\n" % (100))
    print("")
    pickle.dump({'handwriting_datasets': raw_datasets,
                 'formula_id2latex': loaded['formula_id2latex'],
                 'preprocessing_queue': preprocessing_queue},
                open(outputpath, "wb"),
                2)
577871
def update(self, forecasts, observations):
        
        if len(observations.shape) == 1:
            obs_cdfs = np.zeros((observations.size, self.thresholds.size))
            for o, observation in enumerate(observations):
                obs_cdfs[o, self.thresholds >= observation] = 1
        else:
            obs_cdfs = observations
        self.errors["F_2"] += np.sum(forecasts ** 2, axis=0)
        self.errors["F_O"] += np.sum(forecasts * obs_cdfs, axis=0)
        self.errors["O_2"] += np.sum(obs_cdfs ** 2, axis=0)
        self.errors["O"] += np.sum(obs_cdfs, axis=0)
        self.num_forecasts += forecasts.shape[0]
961662
def dataframes(self):
        
        frames = []
        for team in self.__iter__():
            frames.append(team.dataframe)
        return pd.concat(frames)
717298
def loadInputs(self, filename, cols = None, everyNrows = 1,
                   delim = ' ', checkEven = 1):
        
        self.loadInputsFromFile(filename, cols, everyNrows,
                                delim, checkEven)
449193
def __load(arff):
    
    attrs = arff['attributes']
    attrs_t = []
    for attr in attrs:
        if isinstance(attr[1], list):
            attrs_t.append("%s@{%s}" % (attr[0], ','.join(attr[1])))
        else:
            attrs_t.append("%s@%s" % (attr[0], attr[1]))

    df = pd.DataFrame(data=arff['data'], columns=attrs_t)
    return df
172715
def _load_data(self,
                   equities,
                   futures,
                   exchanges,
                   root_symbols,
                   equity_supplementary_mappings):
        
        # Set named identifier columns as indices, if provided.
        _normalize_index_columns_in_place(
            equities=equities,
            equity_supplementary_mappings=equity_supplementary_mappings,
            futures=futures,
            exchanges=exchanges,
            root_symbols=root_symbols,
        )

        futures_output = self._normalize_futures(futures)

        equity_supplementary_mappings_output = (
            self._normalize_equity_supplementary_mappings(
                equity_supplementary_mappings,
            )
        )

        exchanges_output = _generate_output_dataframe(
            data_subset=exchanges,
            defaults=_exchanges_defaults,
        )

        equities_output, equities_mappings = self._normalize_equities(
            equities,
            exchanges_output,
        )

        root_symbols_output = _generate_output_dataframe(
            data_subset=root_symbols,
            defaults=_root_symbols_defaults,
        )

        return AssetData(
            equities=equities_output,
            equities_mappings=equities_mappings,
            futures=futures_output,
            exchanges=exchanges_output,
            root_symbols=root_symbols_output,
            equity_supplementary_mappings=equity_supplementary_mappings_output,
        )
1165459
def forecast(self, horizon, **kwargs):
        ''''''
        return DataFrame({ name: series.forecast(horizon, **kwargs) \
            for name, series in self.groups.iteritems() })
402264
def loadcsv(filename):
    
    dataframe = _pd.read_csv(filename)
    data = {}
    for key, value in dataframe.items():
        data[key] = value.values
    return data
574623
def load(self):
        
        s = load_stack(self)
        if s:
            self.hooks = s.hooks
            self.calls = s.calls
569633
def load_stream(self, stream):
        
        batches = super(ArrowStreamPandasSerializer, self).load_stream(stream)
        import pyarrow as pa
        for batch in batches:
            yield [self.arrow_to_pandas(c) for c in pa.Table.from_batches([batch]).itercolumns()]
646293
def loadings(self):
        """"""
        loadings = self.v[:, : self.keep] * np.sqrt(self.eigenvalues)
        cols = ["PC%s" % i for i in range(1, self.keep + 1)]
        loadings = pd.DataFrame(
            loadings, columns=cols, index=self.feature_names
        )
        return loadings
1311405
def load_data(self):
        
        for m, member in enumerate(self.members):
            mo = ModelOutput(self.ensemble_name, member, self.run_date, self.variable,
                             self.start_date, self.end_date, self.path, self.map_file, self.single_step)
            mo.load_data()
            if self.data is None:
                self.data = np.zeros((len(self.members), mo.data.shape[0], mo.data.shape[1], mo.data.shape[2]),
                                     dtype=np.float32)
            if mo.units == "m":
                self.data[m] = mo.data * 1000
                self.units = "mm"
            else:
                self.data[m] = mo.data
            if self.units == "":
                self.units = mo.units
            del mo.data
            del mo