This loads dataset info from three package files:

    vega_datasets/datasets.json
    vega_datasets/dataset_info.json
    vega_datasets/local_datasets.json

    It returns a dictionary with dataset information.
322473
def _load_dataset_info():
    
    def load_json(path):
        raw = pkgutil.get_data('vega_datasets', path)
        return json.loads(bytes_decode(raw))

    info = load_json('datasets.json')
    descriptions = load_json('dataset_info.json')
    local_datasets = load_json('local_datasets.json')

    for name in info:
        info[name]['is_local'] = (name in local_datasets)
    for name in descriptions:
        info[name].update(descriptions[name])

    return info
1212169
def get_dataset(self, key, info, out=None):
        """"""

        logger.debug("Reading %s.", key.name)
        values = self.file_content[key.name]
        selected = np.array(self.selected)
        if key.name in ("Latitude", "Longitude"):
            values = values / 10000.
        if key.name in ('Tsurf', 'CloudTopPres', 'CloudTopTemp'):
            goods = values > -9998.
            selected = np.array(selected & goods)
            if key.name in ('Tsurf', "Alt_surface", "CloudTopTemp"):
                values = values / 100.
            if key.name in ("CloudTopPres"):
                values = values / 10.
        else:
            selected = self.selected
        info.update(self.finfo)

        fill_value = np.nan

        if key.name == 'ct':
            fill_value = 0
            info['_FillValue'] = 0
        ds = DataArray(values, dims=['y', 'x'], attrs=info).where(selected, fill_value)

        # update dataset info with file_info
        return ds
467660
def datasets(ctx, client):
    """"""
    from renku.models._jsonld import asjsonld
    from renku.models.datasets import Dataset
    from renku.models.refs import LinkReference

    from ._checks.location_datasets import _dataset_metadata_pre_0_3_4

    for old_path in _dataset_metadata_pre_0_3_4(client):
        with old_path.open('r') as fp:
            dataset = Dataset.from_jsonld(yaml.safe_load(fp))

        name = str(old_path.parent.relative_to(client.path / 'data'))
        new_path = (
            client.renku_datasets_path / dataset.identifier.hex /
            client.METADATA
        )
        new_path.parent.mkdir(parents=True, exist_ok=True)

        dataset = dataset.rename_files(
            lambda key: os.path.
            relpath(str(old_path.parent / key), start=str(new_path.parent))
        )

        with new_path.open('w') as fp:
            yaml.dump(asjsonld(dataset), fp, default_flow_style=False)

        old_path.unlink()

        LinkReference.create(client=client,
                             name='datasets/' + name).set_reference(new_path)
379466
def config_dict(name: str) -> Dict[str, Any]:
    
    try:
        content = resource_string(PACKAGE, DATADIR.format(name)).decode()
    except DistributionNotFound as error:
        LOGGER.warning("Cannot load %s from packages: %s", name, error)
        content = DATA_FALLBACK.joinpath(name).read_text()

    return cast(Dict[str, Any], json.loads(content))
19792
def load_info_df(self, file_name=None):
        """"""

        if file_name is None:
            file_name = self.info_file

        with open(file_name, 'r') as infile:
            top_level_dict = json.load(infile)

        new_info_df_dict = top_level_dict['info_df']
        new_info_df = pd.DataFrame(new_info_df_dict)
        self.info_df = new_info_df

        self._prm_packer(top_level_dict['metadata'])
        self.info_file = file_name
        logger.debug("loaded info_df")
        logger.debug(" info_file: %s" % self.info_file)
105451
def gcs_dataset_info_files(dataset_dir):
  """"""
  prefix = posixpath.join(GCS_DATASET_INFO_DIR, dataset_dir, "")
  # Filter for this dataset
  filenames = [el for el in gcs_files(prefix_filter=prefix)
               if el.startswith(prefix) and len(el) > len(prefix)]
  return filenames
147165
def get_help_datapacks(module_name, server_prefix):
    

    _dir = os.path.realpath(
        os.path.join(os.getcwd(), os.path.dirname(__file__)))
    module_dir = "{}/../{}".format(_dir, module_name, "_help.json")
    if os.path.isdir(module_dir):
        module_help_path = "{}/{}".format(module_dir, "_help.json")

        if os.path.isfile(module_help_path):
            return helptools.get_help_datapacks(module_help_path, server_prefix)
        else:
            return [("Help", "{} does not have a help.json file".format(module_name), False)]
    else:
        return [("Help", "No module found called {}".format(module_name), False)]
1088703
def download_data_dictionary(request, dataset_id):
    
    dataset = Dataset.objects.get(pk=dataset_id)
    dataDict = dataset.data_dictionary
    fields = DataDictionaryField.objects.filter(
        parent_dict=dataDict
    ).order_by('columnIndex')

    response = HttpResponse(content_type='text/csv')
    csvName = slugify(dataset.title + ' data dict') + '.csv'
    response['Content-Disposition'] = 'attachment; filename=%s' % (csvName)

    csvWriter = writer(response)
    metaHeader = [
        'Data Dictionary for {0} prepared by {1}'.format(
            dataset.title,
            dataset.uploaded_by
        )
    ]
    csvWriter.writerow(metaHeader)
    trueHeader = ['Column Index', 'Heading', 'Description', 'Data Type']
    csvWriter.writerow(trueHeader)

    for field in fields:
        mappedIndex = field.COLUMN_INDEX_CHOICES[field.columnIndex-1][1]
        csvWriter.writerow(
            [mappedIndex, field.heading, field.description, field.dataType]
        )

    return response
994028
def _infodict(cls, name):
        """"""
        info = cls._dataset_info.get(name, None)
        if info is None:
            raise ValueError('No such dataset {0} exists, '
                             'use list_datasets() to get a list '
                             'of available datasets.'.format(name))
        return info
1201976
def _get_grid_files(self):
        """"""
        grid_file_paths = self.grid_file_paths
        datasets = []
        if isinstance(grid_file_paths, str):
            grid_file_paths = [grid_file_paths]
        for path in grid_file_paths:
            try:
                ds = xr.open_dataset(path, decode_times=False)
            except (TypeError, AttributeError):
                ds = xr.open_mfdataset(path, decode_times=False).load()
            except (RuntimeError, OSError) as e:
                msg = str(e) + ': {}'.format(path)
                raise RuntimeError(msg)
            datasets.append(ds)
        return tuple(datasets)
1042927
def load_dataset(*args, **kwargs):
    
    warnings.warn(
        "load_dataset` will be removed in a future version of xarray. The "
        "current behavior of this function can be achived by using "
        "`tutorial.open_dataset(...).load()`.",
        DeprecationWarning, stacklevel=2)
    return open_dataset(*args, **kwargs).load()
293258
function loadInfo() {
    try {
      var data = fs.readFileSync(__rootdir + '/package.json', { encoding: 'utf8' });
      var tmp = JSON.parse(data);
      appPackage = {
        name: tmp.name || null,
        version: tmp.version || 1
      }
    } catch(e) {
      return; // nothing to do.
    }
  }
1090002
def load(self, d3mds):
        """"""
        X, y = d3mds.get_data()

        return Dataset(d3mds.dataset_id, X, y)
810822
def load_ds_ids_from_config(self):
        """"""
        ids = []
        for dataset in self.datasets.values():
            # xarray doesn't like concatenating attributes that are lists
            # https://github.com/pydata/xarray/issues/2060
            if 'coordinates' in dataset and \
                    isinstance(dataset['coordinates'], list):
                dataset['coordinates'] = tuple(dataset['coordinates'])
            # Build each permutation/product of the dataset
            id_kwargs = []
            for key in DATASET_KEYS:
                val = dataset.get(key)
                if key in ["wavelength", "modifiers"] and isinstance(val,
                                                                     list):
                    # special case: wavelength can be [min, nominal, max]
                    # but is still considered 1 option
                    # it also needs to be a tuple so it can be used in
                    # a dictionary key (DatasetID)
                    id_kwargs.append((tuple(val), ))
                elif key == "modifiers" and val is None:
                    # empty modifiers means no modifiers applied
                    id_kwargs.append((tuple(), ))
                elif isinstance(val, (list, tuple, set)):
                    # this key has multiple choices
                    # (ex. 250 meter, 500 meter, 1000 meter resolutions)
                    id_kwargs.append(val)
                elif isinstance(val, dict):
                    id_kwargs.append(val.keys())
                else:
                    # this key only has one choice so make it a one
                    # item iterable
                    id_kwargs.append((val, ))
            for id_params in itertools.product(*id_kwargs):
                dsid = DatasetID(*id_params)
                ids.append(dsid)

                # create dataset infos specifically for this permutation
                ds_info = dataset.copy()
                for key in DATASET_KEYS:
                    if isinstance(ds_info.get(key), dict):
                        ds_info.update(ds_info[key][getattr(dsid, key)])
                    # this is important for wavelength which was converted
                    # to a tuple
                    ds_info[key] = getattr(dsid, key)
                self.ids[dsid] = ds_info

        return ids
390041
def _read_data(prefix, batch_size, augmentation=None):
    
    prefix = prefix.replace("\\", "/")
    folder = "/".join(prefix.split("/")[:-1])
    phase = prefix.split("/")[-1]
    config = json.load(open(prefix + '_config.json'))
    num_threads = config["num_threads"]

    filenames = [folder + "/" + f for f in listdir(folder) if isfile(join(folder, f)) and phase in f and not "config.json" in f]

    dataset = tf.data.TFRecordDataset(filenames=filenames, num_parallel_reads=num_threads)
    dataset = dataset.shuffle(buffer_size=10 * batch_size)
    dataset = dataset.repeat()
    dataset = dataset.map(map_func=_create_parser_fn(config, phase), num_parallel_calls=num_threads)
    if augmentation is not None:
        dataset = dataset.map(map_func=augmentation, num_parallel_calls=num_threads)
    dataset = dataset.batch(batch_size=batch_size)
    dataset = dataset.prefetch(buffer_size=1)

    return dataset
858765
def _read_datasets(self, dataset_nodes, **kwargs):
        """"""
        # Sort requested datasets by reader
        reader_datasets = {}

        for node in dataset_nodes:
            ds_id = node.name
            # if we already have this node loaded or the node was assigned
            # by the user (node data is None) then don't try to load from a
            # reader
            if ds_id in self.datasets or not isinstance(node.data, dict):
                continue
            reader_name = node.data.get('reader_name')
            if reader_name is None:
                # This shouldn't be possible
                raise RuntimeError("Dependency tree has a corrupt node.")
            reader_datasets.setdefault(reader_name, set()).add(ds_id)

        # load all datasets for one reader at a time
        loaded_datasets = DatasetDict()
        for reader_name, ds_ids in reader_datasets.items():
            reader_instance = self.readers[reader_name]
            new_datasets = reader_instance.load(ds_ids, **kwargs)
            loaded_datasets.update(new_datasets)
        self.datasets.update(loaded_datasets)
        return loaded_datasets
528482
def load_data_file_to_net(self, filename):
    '''
    
    '''
    inst_dat = self.load_json_to_dict(filename)
    load_data.load_data_to_net(self, inst_dat)