Generator which yields a list of valid data directories (corresponds to the content of one .tgz).
423079
def data_folders(path):
        
        for item in os.listdir(path):
            dir_path = os.path.join(path, item)
            wav_folder = os.path.join(dir_path, 'wav')

            if os.path.isdir(dir_path) and os.path.isdir(wav_folder):
                yield dir_path
103595
def glob(self):
        """"""

        # Cached symlinks
        self.symlinks = {}

        if self.is_bytes:
            curdir = os.fsencode(os.curdir)
        else:
            curdir = os.curdir

        for pattern in self.pattern:
            # If the pattern ends with `/` we return the files ending with `/`.
            dir_only = pattern[-1].dir_only if pattern else False

            if pattern:
                if not pattern[0].is_magic:
                    # Path starts with normal plain text
                    # Lets verify the case of the starting directory (if possible)
                    this = pattern[0]

                    curdir = this[0]

                    if not os.path.lexists(curdir):
                        return

                    # Make sure case matches, but running case insensitive
                    # on a case sensitive file system may return more than
                    # one starting location.
                    results = [curdir] if this.is_drive else self._get_starting_paths(curdir)
                    if not results:
                        if not dir_only:
                            # There is no directory with this name,
                            # but we have a file and no directory restriction
                            yield curdir
                        return

                    if this.dir_only:
                        # Glob these directories if they exists
                        for start in results:
                            if os.path.isdir(start):
                                rest = pattern[1:]
                                if rest:
                                    this = rest.pop(0)
                                    for match in self._glob(curdir, this, rest):
                                        if not self._is_excluded(match, dir_only):
                                            yield os.path.join(match, self.empty) if dir_only else match
                                elif not self._is_excluded(curdir, dir_only):
                                    yield os.path.join(curdir, self.empty) if dir_only else curdir
                    else:
                        # Return the file(s) and finish.
                        for start in results:
                            if os.path.lexists(start) and not self._is_excluded(start, dir_only):
                                yield os.path.join(start, self.empty) if dir_only else start
                else:
                    # Path starts with a magic pattern, let's get globbing
                    rest = pattern[:]
                    this = rest.pop(0)
                    for match in self._glob(curdir if not curdir == self.current else self.empty, this, rest):
                        if not self._is_excluded(match, dir_only):
                            yield os.path.join(match, self.empty) if dir_only else match
326999
def generate_data(self, data_dir, tmp_dir, task_id=-1):
    """"""
    filepath_fns = {
        problem.DatasetSplit.TRAIN: self.training_filepaths,
        problem.DatasetSplit.EVAL: self.dev_filepaths,
        problem.DatasetSplit.TEST: self.test_filepaths,
    }

    # We set shuffled=True as we don't want to shuffle on disk later.
    split_paths = [(split["split"], filepath_fns[split["split"]](
        data_dir, split["shards"], shuffled=True))
                   for split in self.dataset_splits]
    all_paths = []
    for _, paths in split_paths:
      all_paths.extend(paths)

    if self.is_generate_per_split:
      for split, paths in split_paths:
        generator_utils.generate_files(
            self.generate_encoded_samples(data_dir, tmp_dir, split),
            paths,
            cycle_every_n=self.total_number_of_frames // len(paths))
    else:
      generator_utils.generate_files(
          self.generate_encoded_samples(data_dir, tmp_dir,
                                        problem.DatasetSplit.TRAIN),
          all_paths,
          cycle_every_n=self.total_number_of_frames // len(all_paths))
223287
def glob1(self, dir_relpath, glob):
    """"""
    if self.isignored(dir_relpath, directory=True):
      return []

    matched_files = self._glob1_raw(dir_relpath, glob)
    prefix = self._relpath_no_dot(dir_relpath)
    return self._filter_ignored(matched_files, selector=lambda p: os.path.join(prefix, p))
173055
def directory_files(path):
    """"""

    for entry in os.scandir(path):
        if not entry.name.startswith('.') and entry.is_file():
            yield entry.name
14448
def prepare_data_maybe_download(directory):
    
    filename = 'ubuntu_dialogs.tgz'
    url = 'http://cs.mcgill.ca/~jpineau/datasets/ubuntu-corpus-1.0/ubuntu_dialogs.tgz'
    dialogs_path = os.path.join(directory, 'dialogs')

    # test it there are some dialogs in the path
    if not os.path.exists(os.path.join(directory, "10", "1.tst")):
        # dialogs are missing
        archive_path = os.path.join(directory, filename)
        if not os.path.exists(archive_path):
            # archive missing, download it
            print("Downloading %s to %s" % (url, archive_path))
            filepath, _ = urllib.request.urlretrieve(url, archive_path)
            print "Successfully downloaded " + filepath

        # unpack data
        if not os.path.exists(dialogs_path):
            print("Unpacking dialogs ...")
            with tarfile.open(archive_path) as tar:
                tar.extractall(path=directory)
            print("Archive unpacked.")

        return
830479
def _get_contents(self):
        """"""
        def files():
            for value in super(GlobBundle, self)._get_contents():
                for path in glob.glob(value):
                    yield path
        return list(files())
471825
def iterfiles(self, pattern=None, abspath=False):
        
        if pattern is not None:
            globster = Globster([pattern])
        for root, dirs, files in self.walk():
            for f in files:
                if pattern is None or (pattern is not None and globster.match(f)):
                    if abspath:
                        yield os.path.join(root, f)
                    else:
                        yield self.relpath(os.path.join(root, f))
749740
def get_data_dirs(__pkg: str) -> List[str]:
    
    dirs = [user_data(__pkg), ]
    dirs.extend(path.expanduser(path.sep.join([d, __pkg]))
                for d in getenv('XDG_DATA_DIRS',
                                '/usr/local/share/:/usr/share/').split(':'))
    return [d for d in dirs if path.isdir(d)]
1017950
def glob_in_parents(dir, patterns, upper_limit=None):
    """"""

    assert(isinstance(dir, str))
    assert(isinstance(patterns, list))

    result = []

    absolute_dir = os.path.join(os.getcwd(), dir)
    absolute_dir = os.path.normpath(absolute_dir)
    while absolute_dir:
        new_dir = os.path.split(absolute_dir)[0]
        if new_dir == absolute_dir:
            break
        result = glob([new_dir], patterns)
        if result:
            break
        absolute_dir = new_dir

    return result
176764
def _get_vqa_v2_image_raw_dataset(directory, image_root_url, image_urls):
  """"""
  for url in image_urls:
    filename = os.path.basename(url)
    download_url = os.path.join(image_root_url, url)
    path = generator_utils.maybe_download(directory, filename, download_url)
    unzip_dir = os.path.join(directory, filename.strip(".zip"))
    if not tf.gfile.Exists(unzip_dir):
      zipfile.ZipFile(path, "r").extractall(directory)
1274639
def _generate_output_dir(settings, path):
        

        relpath = os.path.relpath(settings.root,path)
        count = relpath.count(os.sep) + 1

        return relpath+os.path.sep, count
402455
def list_dataset_uris(cls, base_uri, config_path):
        """"""

        parsed_uri = generous_parse_uri(base_uri)
        uri_list = []

        path = parsed_uri.path
        if IS_WINDOWS:
            path = unix_to_windows_path(parsed_uri.path, parsed_uri.netloc)

        for d in os.listdir(path):
            dir_path = os.path.join(path, d)

            if not os.path.isdir(dir_path):
                continue

            storage_broker = cls(dir_path, config_path)

            if not storage_broker.has_admin_metadata():
                continue

            uri = storage_broker.generate_uri(
                name=d,
                uuid=None,
                base_uri=base_uri
            )
            uri_list.append(uri)

        return uri_list
1141125
def prepare_to_generate(self, data_dir, tmp_dir):
    """"""
    self.get_or_create_vocab(data_dir, tmp_dir)
    self.train_text_filepaths(tmp_dir)
    self.dev_text_filepaths(tmp_dir)
804778
def _next_dir_gen(self, root):
    
    for e in root.getiterator(common._T_COMMON_PREFIXES):
      yield common.GCSFileStat(
          self._path + '/' + e.find(common._T_PREFIX).text,
          st_size=None, etag=None, st_ctime=None, is_dir=True)
      e.clear()
    yield None
724150
def _get_cifar(directory, url):
  """"""
  filename = os.path.basename(url)
  path = generator_utils.maybe_download(directory, filename, url)
  tarfile.open(path, "r:gz").extractall(directory)
1107101
def _parseDataDirectory(self, data, sections, imageNtHeaders):
        """"""
        data_directory_data_list = [None for i in range(15)]

        # parse DataDirectory[Export]
        export_data_directory = imageNtHeaders.header.OptionalHeader.DataDirectory[ImageDirectoryEntry.EXPORT]
        export_section = self._getSectionForDataDirectoryEntry(export_data_directory, sections)
        export_data_directory_data = self._parseDataDirectoryExport(data, export_data_directory, export_section)
        data_directory_data_list[ImageDirectoryEntry.EXPORT] = export_data_directory_data

        # parse DataDirectory[Import]
        import_data_directory = imageNtHeaders.header.OptionalHeader.DataDirectory[ImageDirectoryEntry.IMPORT]
        import_section = self._getSectionForDataDirectoryEntry(import_data_directory, sections)
        import_data_directory_data = self._parseDataDirectoryImport(import_data_directory, import_section)
        data_directory_data_list[ImageDirectoryEntry.IMPORT] = import_data_directory_data

        # parse DataDirectory[LOAD_CONFIG]
        loadconfig_data_directory = imageNtHeaders.header.OptionalHeader.DataDirectory[ImageDirectoryEntry.LOAD_CONFIG]
        loadconfig_section = self._getSectionForDataDirectoryEntry(loadconfig_data_directory, sections)
        loadconfig_data = self._parseLoadConfig(loadconfig_data_directory, loadconfig_section)
        data_directory_data_list[ImageDirectoryEntry.LOAD_CONFIG] = loadconfig_data

        return data_directory_data_list