Generate examples for the Shapes3d dataset.

    Args:
      filepath: path to the Shapes3d hdf5 file.

    Yields:
      Dictionaries with images and the different labels.
332807
def _generate_examples(self, filepath):
    
    # Simultaneously iterating through the different data sets in the hdf5
    # file will be slow with a single file. Instead, we first load everything
    # into memory before yielding the samples.
    image_array, values_array = _load_data(filepath)

    # We need to calculate the class labels from the float values in the file.
    labels_array = np.zeros_like(values_array, dtype=np.int64)
    for i in range(values_array.shape[1]):
      labels_array[:, i] = _discretize(values_array[:, i])  # pylint: disable=unsupported-assignment-operation

    for image, labels, values in moves.zip(image_array, labels_array,
                                           values_array):
      yield {
          "image": image,
          "label_floor_hue": labels[0],
          "label_wall_hue": labels[1],
          "label_object_hue": labels[2],
          "label_scale": labels[3],
          "label_shape": labels[4],
          "label_orientation": labels[5],
          "value_floor_hue": values[0],
          "value_wall_hue": values[1],
          "value_object_hue": values[2],
          "value_scale": values[3],
          "value_shape": values[4],
          "value_orientation": values[5],
      }
710535
def iterSourceCode(paths):
    
    for path in paths:
        if os.path.isdir(path):
            for dirpath, dirnames, filenames in os.walk(path):
                for filename in filenames:
                    full_path = os.path.join(dirpath, filename)
                    if isPythonFile(full_path):
                        yield full_path
        else:
            yield path
1311817
def _generate_examples(self, data_dir):
    
    corruption = self.builder_config.corruption
    severity = self.builder_config.severity

    images_file = os.path.join(data_dir, _CORRUPTIONS_TO_FILENAMES[corruption])
    labels_file = os.path.join(data_dir, _LABELS_FILENAME)

    with tf.io.gfile.GFile(labels_file, mode='rb') as f:
      labels = np.load(f)

    num_images = labels.shape[0] // 5
    # Labels are stacked 5 times so we can just read the first iteration
    labels = labels[:num_images]

    with tf.io.gfile.GFile(images_file, mode='rb') as f:
      images = np.load(f)

    # Slice images corresponding to correct severity level
    images = images[(severity - 1) * num_images:severity * num_images]

    for image, label in zip(images, labels):
      yield {
          'image': image,
          'label': label,
      }
876901
def load_hdf(cls, filename, path='', ic=None):
        
        store = pd.HDFStore(filename)
        try:
            samples = store[path+'/df']
            attrs = store.get_storer(path+'/df').attrs
        except:
            store.close()
            raise
        df = store[path+'/df']
        new = cls.from_df(df)

        if ic is None:
            ic = get_ichrone('mist')

        new.define_models(ic, N=attrs.N, index=attrs.index)
        new.spectroscopy = attrs.spectroscopy
        new.parallax = attrs.parallax
        store.close()
        return new
518968
def _loadfilepath(self, filepath, **kwargs):
        
        with open(filepath, "r") as f:
            data = json.load(f, **kwargs)
        return data
326999
def generate_data(self, data_dir, tmp_dir, task_id=-1):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    # We set shuffled=True as we don't want to shuffle on disk later.
    split_paths = [(split["split"], filepath_fns[split["split"]](
        data_dir, split["shards"], shuffled=True))
                   for split in self.dataset_splits]
    all_paths = []
    for _, paths in split_paths:
      all_paths.extend(paths)

    if self.is_generate_per_split:
      for split, paths in split_paths:
        generator_utils.generate_files(
            self.generate_encoded_samples(data_dir, tmp_dir, split),
            paths,
            cycle_every_n=self.total_number_of_frames // len(paths))
    else:
      generator_utils.generate_files(
          self.generate_encoded_samples(data_dir, tmp_dir,
                                        problem.DatasetSplit.TRAIN),
          all_paths,
          cycle_every_n=self.total_number_of_frames // len(all_paths))
1226633
def __generate_file(self, file_path):
        
        file_path = os.path.expanduser(file_path)
        if not os.path.exists(os.path.dirname(file_path)):
            self.logger.debug("Directories missing! Creating directories for %s..." % file_path)
            os.makedirs(os.path.dirname(file_path))
        if not os.path.exists(file_path):
            open(file_path, "w+").close()
        return file_path
1321103
def _generate_examples(self, num_examples, data_path, label_path):
    
    images = _extract_mnist_images(data_path, num_examples)
    labels = _extract_mnist_labels(label_path, num_examples)
    data = list(zip(images, labels))

    # Data is shuffled automatically to distribute classes uniformly.
    for image, label in data:
      yield {
          "image": image,
          "label": label,
      }
1316939
def dict_to_hdf5(dic, endpoint):
    
    filename = gen_filename(endpoint)
    with h5py.File(filename, 'w') as handler:
        walk_dict_to_hdf5(dic, handler)
    print('dumped to', filename)
1300260
def _generate_examples(self, label_images):
    """"""

    for label, image_paths in label_images.items():
      for image_path in image_paths:
        yield {
            "image": image_path,
            "label": label,
        }
553092
def iter_doc_filepaths(self, **kwargs):  # pylint: disable=W0613
        
        with self._index_lock:
            for doc_id, info in self._doc_index.items():
                if not self._is_alias(doc_id):
                    yield doc_id, info[-1]
739416
def _generate_examples(self, archive):
    """"""
    num_skipped = 0
    for fname, fobj in archive:
      res = _NAME_RE.match(fname)
      if not res:  # README file, ...
        continue
      label = res.group(1).lower()
      if tf.compat.as_bytes("JFIF") not in fobj.peek(10):
        num_skipped += 1
        continue
      yield {
          "image": fobj,
          "image/filename": fname,
          "label": label,
      }

    if num_skipped != _NUM_CORRUPT_IMAGES:
      raise ValueError("Expected %d corrupt images, but found %d" % (
          _NUM_CORRUPT_IMAGES, num_skipped))
    logging.warning("%d images were corrupted and were skipped", num_skipped)
1008266
def _generate_examples(self, archive_paths, objects_getter, bboxes_getter,
                         prefixes=None):
    """"""
    trainable_classes = set(
        self.info.features['objects_trainable']['label'].names)
    for i, archive_path in enumerate(archive_paths):
      prefix = prefixes[i] if prefixes else None
      objects = objects_getter(prefix)
      bboxes = bboxes_getter(prefix)
      logging.info('Opening archive %s ...', archive_path)
      archive = tfds.download.iter_archive(
          archive_path, tfds.download.ExtractMethod.TAR_STREAM)
      for fpath, fobj in archive:
        fname = os.path.basename(fpath)
        image_id = int(os.path.splitext(fname)[0], 16)
        image_objects = [obj._asdict() for obj in objects.get(image_id, [])]
        image_bboxes = [bbox._asdict() for bbox in bboxes.get(image_id, [])]
        image_objects_trainable = [
            obj for obj in image_objects if obj['label'] in trainable_classes
        ]
        yield {
            'image': _resize_image_if_necessary(
                fobj, target_pixels=self.builder_config.target_pixels),
            'image/filename': fname,
            'objects': image_objects,
            'objects_trainable': image_objects_trainable,
            'bobjects': image_bboxes,
        }
878820
def gen_trace_planes(self, ridx):
        
        with h5py.File(self.source_file, "r") as hdf5:
            for idx in ridx:
                trace = "{:s}/{:s}".format(self.idx_set["sec"], str(idx))
                plane = hdf5[trace + "/RupturePlanes"][:].astype("float64")
                yield trace, plane
395504
def get_help_datapacks(filepath, prefix="!"):
    

    help_contents = get_help_data(filepath)

    datapacks = []

    # Add the content
    for d in help_contents:
        heading = d
        content = ""

        if "commands" in d.lower():
            for c in help_contents[d]:
                if "name" not in c:
                    continue

                content += "- `"
                command = prefix + c["name"]
                content += "{}".format(command)
                if "params" in c:
                    for param in c["params"]:
                        content += " [{}]".format(param)
                content += "`: "
                if "description" in c:
                    content += c["description"]
                content += "\n"
        else:
            content += help_contents[d]

        datapacks.append((heading, content, False))

    return datapacks
881835
def __ReadFromFile(self, datapath):
        
        def _create_meta(_datapath):
            """Just simply returns some dict. This functions exists in order to keep DRY"""
            meta = {
                'series_number': 0,
                'datadir': _datapath
            }
            return meta

        path, ext = os.path.splitext(datapath)
        ext = ext[1:]
        if ext in ('pklz', 'pkl'):
            logger.debug('pklz format detected')
            from . import misc
            data = misc.obj_from_file(datapath, filetype='pkl')
            data3d = data.pop('data3d')
            # metadata must have series_number
            metadata = _create_meta(datapath)
            metadata.update(data)

        elif ext in ['hdf5']:
            from . import hdf5_io
            datap = hdf5_io.load_dict_from_hdf5(datapath)
            # datap = self.read_hdf5(datapath)
            data3d = datap.pop('data3d')

            # back compatibility
            if 'metadata' in datap.keys():
                datap = datap['metadata']
            # metadata must have series_number
            metadata = _create_meta(datapath)
            metadata.update(datap)

        elif ext in ['idx']:
            from . import idxformat
            idxreader = idxformat.IDXReader()
            data3d, metadata = idxreader.read(datapath)
        elif ext in ['dcm', 'DCM', 'dicom']:
            data3d, metadata = self._read_with_sitk(datapath)
            metadata = self._fix_sitk_bug(datapath, metadata)
        elif ext in ["bz2"]:
            new_datapath = tgz.untar(datapath)
            data3d, metadata = self.__ReadFromDirectory(new_datapath)
        else:
            logger.debug('file format "' + str(ext) + '"')
            # reading raw file
            data3d, metadata = self._read_with_sitk(datapath)
        return data3d, metadata
75785
def get_filenodes_generator(self):
        
        for topnode, dirs, files in self.walk():
            for node in files:
                yield node