Convert dataset files.
426529
def _convert_dataset_files(value):
    """"""
    output = {}
    for k, v in value.items():
        inst = DatasetFile.from_jsonld(v)
        output[inst.path] = inst
    return output
1009956
def _download_datasets():
    """"""
    def filepath(*args):
        return abspath(join(dirname(__file__), '..', 'vega_datasets', *args))
    dataset_listing = {}
    for name in DATASETS_TO_DOWNLOAD:
        data = Dataset(name)
        url = data.url
        filename = filepath('_data', data.filename)
        print("retrieving data {0} -> {1}".format(url, filename))
        urlretrieve(url, filename)
        dataset_listing[name] = '_data/{0}'.format(data.filename)
    with open(filepath('local_datasets.json'), 'w') as f:
        json.dump(dataset_listing, f, indent=2, sort_keys=True)
1109652
def choose_raw_dataset(currently=""):
    """"""
    folder = os.path.join(get_project_root(), "raw-datasets")
    files = [os.path.join(folder, name) for name in os.listdir(folder)
             if name.endswith(".pickle")]
    default = -1
    for i, filename in enumerate(files):
        if os.path.basename(currently) == os.path.basename(filename):
            default = i
        if i != default:
            print("[%i]\t%s" % (i, os.path.basename(filename)))
        else:
            print("\033[1m[%i]\033[0m\t%s" % (i, os.path.basename(filename)))
    i = input_int_default("Choose a dataset by number: ", default)
    return files[i]
1314337
def parse_csv_dataset(data, handle_units):
    """"""
    fobj = BytesIO(data)
    names, units = parse_csv_header(fobj.readline().decode('utf-8'))
    arrs = np.genfromtxt(fobj, dtype=None, names=names, delimiter=',', unpack=True,
                         converters={'date': lambda s: parse_iso_date(s.decode('utf-8'))})
    d = {}
    for f in arrs.dtype.fields:
        dat = arrs[f]
        if dat.dtype == np.object:
            dat = dat.tolist()
        d[f] = handle_units(dat, units.get(f, None))
    return d
178823
def dataset(directory, images_file, labels_file):
  """"""

  images_file = download(directory, images_file)
  labels_file = download(directory, labels_file)

  check_image_file_header(images_file)
  check_labels_file_header(labels_file)

  def decode_image(image):
    # Normalize from [0, 255] to [0.0, 1.0]
    image = tf.decode_raw(image, tf.uint8)
    image = tf.cast(image, tf.float32)
    image = tf.reshape(image, [784])
    return image / 255.0

  def decode_label(label):
    label = tf.decode_raw(label, tf.uint8)  # tf.string -> [tf.uint8]
    label = tf.reshape(label, [])  # label is a scalar
    return tf.to_int32(label)

  images = tf.data.FixedLengthRecordDataset(
      images_file, 28 * 28, header_bytes=16).map(decode_image)
  labels = tf.data.FixedLengthRecordDataset(
      labels_file, 1, header_bytes=8).map(decode_label)
  return tf.data.Dataset.zip((images, labels))
465789
def _load_dataset(dsid, ds_info, file_handlers, dim='y'):
        """"""
        slice_list = []
        failure = True
        for fh in file_handlers:
            try:
                projectable = fh.get_dataset(dsid, ds_info)
                if projectable is not None:
                    slice_list.append(projectable)
                    failure = False
            except KeyError:
                logger.warning("Failed to load {} from {}".format(dsid, fh),
                               exc_info=True)

        if failure:
            raise KeyError(
                "Could not load {} from any provided files".format(dsid))

        if dim not in slice_list[0].dims:
            return slice_list[0]
        res = xr.concat(slice_list, dim=dim)

        combined_info = file_handlers[0].combine_info(
            [p.attrs for p in slice_list])

        res.attrs = combined_info
        return res
1295739
public static BatchRecord fromDataSet(DataSet dataSet) {
        BatchRecord batchRecord = new BatchRecord();
        for (int i = 0; i < dataSet.numExamples(); i++) {
            batchRecord.add(CSVRecord.fromRow(dataSet.get(i)));
        }

        return batchRecord;
    }
999902
def _load(self, dataset='train'):
        
        data, labels = None, None
        if dataset is 'train':
            files = [os.path.join(self.cifar10_dir, 'data_batch_%d' % i) for i in range(1, 6)]
        else:
            files = [os.path.join(self.cifar10_dir, 'test_batch')]

        for file in files:
            if not os.path.exists(file):
                raise FileNotFoundError('Failed to find file: ' + file)

        # Load the data from the batch files
        for file in files:
            with open(file, 'rb') as f:
                cifar10 = pickle.load(f, encoding='latin1')

            if labels is None:
                labels = np.array(cifar10['labels'])
            else:
                labels = np.concatenate((labels, cifar10['labels']), axis=0)

            if data is None:
                data = cifar10['data']
            else:
                data = np.concatenate((data, cifar10['data']), axis=0)

        # Adapt the format of the data to our convnet
        data = np.array(data, dtype=float) / 255.0
        data = data.reshape([-1, self.num_channels, self.img_size, self.img_size])
        data = data.transpose([0, 2, 3, 1])

        # One-hot encode labels (see https://stackoverflow.com/a/42874726)
        labels = np.eye(self.num_classes)[np.array(labels).reshape(-1)]

        if dataset is 'train':
            self._train_data, self._train_labels = data, labels
        else:
            self._test_data, self._test_labels = data, labels
290064
def _gdal_write_datasets(self, dst_ds, datasets):
        """"""
        for i, band in enumerate(datasets['bands']):
            chn = datasets.sel(bands=band)
            bnd = dst_ds.GetRasterBand(i + 1)
            bnd.SetNoDataValue(0)
            bnd.WriteArray(chn.values)
864671
def dataset(self, **kwargs):
        
        kwargs_copy = self.base_dict.copy()
        kwargs_copy.update(**kwargs)
        self._replace_none(kwargs_copy)
        try:
            return NameFactory.dataset_format.format(**kwargs_copy)
        except KeyError:
            return None
927903
def _read_data(self, dataset, format=None):
        """"""
        # Attempt to detect file format if "format" isn't specified
        if not format:
            format_class = formats.detect(dataset)
        else:
            if format not in formats.AVAILABLE.keys():
                raise ValueError("'{0}' format not supported.".format(format))
            format_class = formats.AVAILABLE[format]
        return format_class(dataset).to_iterable()
1052033
def is_dataset(ds):
  """"""
  import tensorflow as tf
  from tensorflow_datasets.core.utils import py_utils
  dataset_types = [tf.data.Dataset]
  v1_ds = py_utils.rgetattr(tf, "compat.v1.data.Dataset", None)
  v2_ds = py_utils.rgetattr(tf, "compat.v2.data.Dataset", None)
  if v1_ds is not None:
    dataset_types.append(v1_ds)
  if v2_ds is not None:
    dataset_types.append(v2_ds)
  return isinstance(ds, tuple(dataset_types))
417458
def to_dict(self, trim=True):
        """"""
        if trim:
            return self._to_trimmed_dict()
        else:
            return dict(zip(DATASET_KEYS, self))
1256354
def save_datasets(self, writer="geotiff", datasets=None, compute=True, **kwargs):
        """"""
        if datasets is not None:
            datasets = [self[ds] for ds in datasets]
        else:
            datasets = [self.datasets.get(ds) for ds in self.wishlist]
            datasets = [ds for ds in datasets if ds is not None]
        if not datasets:
            raise RuntimeError("None of the requested datasets have been "
                               "generated or could not be loaded. Requested "
                               "composite inputs may need to have matching "
                               "dimensions (eg. through resampling).")
        writer, save_kwargs = load_writer(writer, ppp_config_dir=self.ppp_config_dir, **kwargs)
        return writer.save_datasets(datasets, compute=compute, **save_kwargs)
767907
def get_dataset(dataset_name, dataset_prefix=None, data_dir=None,
                url=None, resume=True, verbose=1):
    
    warn(DEPRECATION_MSG)
    file_id, md5 = OSF_RESOURCES[dataset_name]
    if url is None:
        url = '{}/{}'.format(OSF_PROJECT_URL, file_id)
    return fetch_file(dataset_name, url, data_dir, dataset_prefix=dataset_prefix,
                      filetype='tar', resume=resume, verbose=verbose, md5sum=md5)
932212
def run(self, dataset_path):
        """"""
        features = self._generate_features(self._feature_extractors)
        features.to_csv(dataset_path)
795951
def dump_to_store(dataset, store, writer=None, encoder=None,
                  encoding=None, unlimited_dims=None):
    """"""
    if writer is None:
        writer = ArrayWriter()

    if encoding is None:
        encoding = {}

    variables, attrs = conventions.encode_dataset_coordinates(dataset)

    check_encoding = set()
    for k, enc in encoding.items():
        # no need to shallow copy the variable again; that already happened
        # in encode_dataset_coordinates
        variables[k].encoding = enc
        check_encoding.add(k)

    if encoder:
        variables, attrs = encoder(variables, attrs)

    store.store(variables, attrs, check_encoding, writer,
                unlimited_dims=unlimited_dims)