tries to autocreate a dataset from images in the given directory
334545
def create_dset(directory,slice_order='alt+z',sort_order='zt',force_slices=None):
    ''''''
    return _create_dset_dicom(directory,slice_order,sort_order,force_slices=force_slices)
1008266
def _generate_examples(self, archive_paths, objects_getter, bboxes_getter,
                         prefixes=None):
    """"""
    trainable_classes = set(
        self.info.features['objects_trainable']['label'].names)
    for i, archive_path in enumerate(archive_paths):
      prefix = prefixes[i] if prefixes else None
      objects = objects_getter(prefix)
      bboxes = bboxes_getter(prefix)
      logging.info('Opening archive %s ...', archive_path)
      archive = tfds.download.iter_archive(
          archive_path, tfds.download.ExtractMethod.TAR_STREAM)
      for fpath, fobj in archive:
        fname = os.path.basename(fpath)
        image_id = int(os.path.splitext(fname)[0], 16)
        image_objects = [obj._asdict() for obj in objects.get(image_id, [])]
        image_bboxes = [bbox._asdict() for bbox in bboxes.get(image_id, [])]
        image_objects_trainable = [
            obj for obj in image_objects if obj['label'] in trainable_classes
        ]
        yield {
            'image': _resize_image_if_necessary(
                fobj, target_pixels=self.builder_config.target_pixels),
            'image/filename': fname,
            'objects': image_objects,
            'objects_trainable': image_objects_trainable,
            'bobjects': image_bboxes,
        }
971209
def get_dataset(self, dataset_id, ds_info, xslice=slice(None), yslice=slice(None)):
        """"""
        var_path = ds_info.get('file_key', '{}'.format(dataset_id.name))
        metadata = self.get_metadata(dataset_id, ds_info)
        shape = metadata['shape']
        file_shape = self[var_path + '/shape']
        if isinstance(shape, tuple) and len(shape) == 2:
            # 2D array
            if xslice.start is not None:
                shape = (shape[0], xslice.stop - xslice.start)
            if yslice.start is not None:
                shape = (yslice.stop - yslice.start, shape[1])
        elif isinstance(shape, tuple) and len(shape) == 1 and yslice.start is not None:
            shape = ((yslice.stop - yslice.start) / yslice.step,)
        metadata['shape'] = shape

        valid_min = self[var_path + '/attr/valid_min']
        valid_max = self[var_path + '/attr/valid_max']
        # no need to check fill value since we are using valid min/max
        scale_factor = self.get(var_path + '/attr/scale_factor')
        add_offset = self.get(var_path + '/attr/add_offset')

        if isinstance(file_shape, tuple) and len(file_shape) == 3:
            data = self[var_path][0, yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 2:
            data = self[var_path][yslice, xslice]
        elif isinstance(file_shape, tuple) and len(file_shape) == 1:
            data = self[var_path][yslice]
        else:
            data = self[var_path]
        data = data.where((data >= valid_min) & (data <= valid_max))
        if scale_factor is not None:
            data = data * scale_factor + add_offset

        if ds_info.get('cloud_clear', False):
            # clear-sky if bit 15-16 are 00
            clear_sky_mask = (self['l2p_flags'][0] & 0b1100000000000000) != 0
            data = data.where(~clear_sky_mask)

        data.attrs.update(metadata)
        return data
478850
def _load_dataset_clipping(self, dataset_dir, epsilon):
    
    self.dataset_max_clip = {}
    self.dataset_min_clip = {}
    self._dataset_image_count = 0
    for fname in os.listdir(dataset_dir):
      if not fname.endswith('.png'):
        continue
      image_id = fname[:-4]
      image = np.array(
          Image.open(os.path.join(dataset_dir, fname)).convert('RGB'))
      image = image.astype('int32')
      self._dataset_image_count += 1
      self.dataset_max_clip[image_id] = np.clip(image + epsilon,
                                                0,
                                                255).astype('uint8')
      self.dataset_min_clip[image_id] = np.clip(image - epsilon,
                                                0,
                                                255).astype('uint8')
1311817
def _generate_examples(self, data_dir):
    
    corruption = self.builder_config.corruption
    severity = self.builder_config.severity

    images_file = os.path.join(data_dir, _CORRUPTIONS_TO_FILENAMES[corruption])
    labels_file = os.path.join(data_dir, _LABELS_FILENAME)

    with tf.io.gfile.GFile(labels_file, mode='rb') as f:
      labels = np.load(f)

    num_images = labels.shape[0] // 5
    # Labels are stacked 5 times so we can just read the first iteration
    labels = labels[:num_images]

    with tf.io.gfile.GFile(images_file, mode='rb') as f:
      images = np.load(f)

    # Slice images corresponding to correct severity level
    images = images[(severity - 1) * num_images:severity * num_images]

    for image, label in zip(images, labels):
      yield {
          'image': image,
          'label': label,
      }
1289231
def images(self):
        """"""
        for ds_id, projectable in self.datasets.items():
            if ds_id in self.wishlist:
                yield projectable.to_image()
1107476
def create(cls, train_ds, valid_ds, test_ds=None, path:PathOrStr='.', no_check:bool=False, bs=64, val_bs:int=None,
               num_workers:int=0, device:torch.device=None, collate_fn:Callable=data_collate,
               dl_tfms:Optional[Collection[Callable]]=None, bptt:int=70, backwards:bool=False, **dl_kwargs) -> DataBunch:
        ""
        datasets = cls._init_ds(train_ds, valid_ds, test_ds)
        val_bs = ifnone(val_bs, bs)
        datasets = [LanguageModelPreLoader(ds, shuffle=(i==0), bs=(bs if i==0 else val_bs), bptt=bptt, backwards=backwards)
                    for i,ds in enumerate(datasets)]
        val_bs = bs
        dls = [DataLoader(d, b, shuffle=False, **dl_kwargs) for d,b in zip(datasets, (bs,val_bs,val_bs,val_bs)) if d is not None]
        return cls(*dls, path=path, device=device, dl_tfms=dl_tfms, collate_fn=collate_fn, no_check=no_check)
427678
def cli():
    """"""
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "dataset_directory",
        help="Directory with datasets to be served"
    )
    parser.add_argument(
        "-p",
        "--port",
        type=int,
        default=8081,
        help="Port to serve datasets on (default 8081)"
    )
    args = parser.parse_args()
    if not os.path.isdir(args.dataset_directory):
        parser.error("Not a directory: {}".format(args.dataset_directory))

    serve_dtool_directory(args.dataset_directory, args.port)
193117
def _get_vqa_v2_image_feature_dataset(
    directory, feature_url, feature_filename="mscoco_feat.tar.gz"):
  """"""
  feature_file = generator_utils.maybe_download_from_drive(
      directory, feature_filename, feature_url)
  with tarfile.open(feature_file, "r:gz") as feature_tar:
    feature_tar.extractall(directory)
1119300
def add_the_init(root_dir, data):
  """"""
  init_file = os.path.join(root_dir, '{dataset_type}', '__init__.py')
  context = (
      'from tensorflow_datasets.{dataset_type}.{dataset_name} import '
      '{dataset_cls}  # {TODO} Sort alphabetically\n'
  )
  with gfile.GFile(init_file.format(**data), 'a') as f:
    f.write(context.format(**data))
939716
def main(_):
  """"""
  batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]
  for filenames, images in load_images(FLAGS.input_dir, batch_shape):
    save_images(images, filenames, FLAGS.output_dir)
465246
def _scanDataFiles(self, dataDir, patterns):
        
        numDataFiles = 0
        for pattern in patterns:
            scanPath = os.path.join(dataDir, pattern)
            for filename in glob.glob(scanPath):
                self._addDataFile(filename)
                numDataFiles += 1
        if numDataFiles == 0:
            raise exceptions.EmptyDirException(dataDir, patterns)
485886
def _split_generators(self, dl_manager):
    """"""
    path = dl_manager.manual_dir
    train_path = os.path.join(path, _TRAIN_DIR)
    val_path = os.path.join(path, _VALIDATION_DIR)

    if not tf.io.gfile.exists(train_path) or not tf.io.gfile.exists(val_path):
      msg = ("You must download the dataset folder from CheXpert"
             "website manually and place it into %s." % path)
      raise AssertionError(msg)

    return [
        tfds.core.SplitGenerator(
            name=tfds.Split.TRAIN,
            num_shards=100,
            gen_kwargs={
                "imgs_path": path,  # Relative img path is provided in csv
                "csv_path": os.path.join(path, _TRAIN_LABELS_FNAME)
            },
        ),
        tfds.core.SplitGenerator(
            name=tfds.Split.VALIDATION,
            num_shards=10,
            gen_kwargs={
                "imgs_path": path,
                "csv_path": os.path.join(path, _VALIDATION_LABELS_FNAME)
            },
        ),
    ]
1165471
def main(_):
  """"""
  eps = FLAGS.max_epsilon / 255.0
  batch_shape = [FLAGS.batch_size, FLAGS.image_height, FLAGS.image_width, 3]

  with tf.Graph().as_default():
    x_input = tf.placeholder(tf.float32, shape=batch_shape)
    noisy_images = x_input + eps * tf.sign(tf.random_normal(batch_shape))
    x_output = tf.clip_by_value(noisy_images, 0.0, 1.0)

    with tf.Session(FLAGS.master) as sess:
      for filenames, images in load_images(FLAGS.input_dir, batch_shape):
        out_images = sess.run(x_output, feed_dict={x_input: images})
        save_images(out_images, filenames, FLAGS.output_dir)
1231927
def get_image(dataset):
    """"""
    dim, nrow, ncol = dataset.shape
    uint8_dataset = dataset.astype('uint8')
    if not (uint8_dataset == dataset).all():
        message = (
            "\nYour image was cast to a `uint8` (`<img>.astype(uint8)`), "
            "but some information was lost.\nPlease check your gif and "
            "convert to uint8 beforehand if the gif looks wrong."
        )
        warnings.warn(message)
    image = [[
            struct.pack(
                'BBB',
                uint8_dataset[0, i, j],
                uint8_dataset[1, i, j],
                uint8_dataset[2, i, j]
            )
            for j in range(ncol)]
        for i in range(nrow)]
    return image
1007971
def importData(directory):
    """"""
    dataTask = OrderedDict()
    dataQueue = OrderedDict()
    for fichier in sorted(os.listdir(directory)):
        try:
            with open("{directory}/{fichier}".format(**locals()), 'rb') as f:
                fileName, fileType = fichier.rsplit('-', 1)
                if fileType == "QUEUE":
                    dataQueue[fileName] = pickle.load(f)
                else:
                    dataTask[fileName] = pickle.load(f)
        except:
            # Can be a directory
            pass
    return dataTask, dataQueue
1081523
def fromDirectory(buildDir):
        """"""
        errorCode = c_uint()
        try:
            cdb = conf.lib.clang_CompilationDatabase_fromDirectory(buildDir,
                byref(errorCode))
        except CompilationDatabaseError as e:
            raise CompilationDatabaseError(int(errorCode.value),
                                           "CompilationDatabase loading failed")
        return cdb