Default file path retrieval function.
        sprinter() - Generates file path list using pool processing and Queues
        crawler() - Generates file path list using os.walk() in sequence
333183
def walk(self):
        
        if self.parallelize:
            self.filepaths = Sprinter(self.directory, self.filters, self.full_paths, self.pool_size, self._printer).sprinter()
        else:
            self.filepaths = Crawler(self.directory, self.filters, self.full_paths, self.topdown, self._printer).crawler()
        return self._get_filepaths()
16783
def list(self, path, filename=None, start=None, stop=None, recursive=False, directories=False):
        
        storageScheme, keys = self.getkeys(
            path, filename=filename, directories=directories, recursive=recursive)
        keys = [storageScheme + ":///" + key.bucket.name + "/" + key.name for key in keys]
        keys.sort()
        keys = select(keys, start, stop)
        return keys
188013
def ListDirectory(self, pathspec, depth=0):
    """"""
    # Limit recursion depth
    if depth >= self.request.max_depth:
      return

    try:
      fd = vfs.VFSOpen(pathspec, progress_callback=self.Progress)
      files = fd.ListFiles(ext_attrs=self.request.collect_ext_attrs)
    except (IOError, OSError) as e:
      if depth == 0:
        # We failed to open the directory the server asked for because dir
        # doesn't exist or some other reason. So we set status and return
        # back to the caller ending the Iterator.
        self.SetStatus(rdf_flows.GrrStatus.ReturnedStatus.IOERROR, e)
      else:
        # Can't open the directory we're searching, ignore the directory.
        logging.info("Find failed to ListDirectory for %s. Err: %s", pathspec,
                     e)
      return

    # If we are not supposed to cross devices, and don't know yet
    # which device we are on, we need to find out.
    if not self.request.cross_devs and self.filesystem_id is None:
      dir_stat = fd.Stat()
      self.filesystem_id = dir_stat.st_dev

    for file_stat in files:

      if stat.S_ISDIR(int(file_stat.st_mode)):
        # Do not traverse directories in a different filesystem.
        is_same_fs = self.filesystem_id == file_stat.st_dev
        if is_same_fs or self.request.cross_devs:
          for child_stat in self.ListDirectory(file_stat.pathspec, depth + 1):
            yield child_stat

      yield file_stat
32005
def ftp_listing_paths(ftpconn: FTP, root: str) -> Iterable[str]:
    """"""

    for current_path, dirs, files in ftp_walk(ftpconn, root):
        yield from (os.path.join(current_path, file) for file in files)
65173
def _EntriesGenerator(self):
    
    location = getattr(self.path_spec, 'location', None)

    if location and location.startswith(self._file_system.PATH_SEPARATOR):
      cpio_archive_file = self._file_system.GetCPIOArchiveFile()
      for cpio_archive_file_entry in cpio_archive_file.GetFileEntries(
          path_prefix=location[1:]):

        path = cpio_archive_file_entry.path
        if not path:
          continue

        _, suffix = self._file_system.GetPathSegmentAndSuffix(
            location[1:], path)

        # Ignore anything that is part of a sub directory or the directory
        # itself.
        if suffix or path == location:
          continue

        path_spec_location = self._file_system.JoinPath([path])
        yield cpio_path_spec.CPIOPathSpec(
            location=path_spec_location, parent=self.path_spec.parent)
282771
def list_files(self, prefix, flat=False):
    

    layer_path = self.get_path_to_file("")        
    path = os.path.join(layer_path, prefix)

    @retry
    def s3lst(continuation_token=None):
      kwargs = {
        'Bucket': self._path.bucket,
        'Prefix': path,
      }

      if continuation_token:
        kwargs['ContinuationToken'] = continuation_token

      return self._conn.list_objects_v2(**kwargs)

    resp = s3lst()

    def iterate(resp):
      if 'Contents' not in resp.keys():
        resp['Contents'] = []

      for item in resp['Contents']:
        key = item['Key']
        filename = key.replace(layer_path, '')
        if not flat and filename[-1] != '/':
          yield filename
        elif flat and '/' not in key.replace(path, ''):
          yield filename


    for filename in iterate(resp):
      yield filename

    while resp['IsTruncated'] and resp['NextContinuationToken']:
      resp = s3lst(resp['NextContinuationToken'])

      for filename in iterate(resp):
        yield filename
732788
def file_path(self, request, response=None, info=None):
        
        return json.dumps(self._extract_key_info(request))
554107
def objwalk(self, obj, path=(), memo=None):
        

        # dual python 2/3 compatability, inspired by the "six" library
        string_types = (str, unicode) if str is bytes else (str, bytes)
        iteritems = lambda mapping: getattr(mapping, 'iteritems', mapping.items)()

        if memo is None:
            memo = set()
        iterator = None
        if isinstance(obj, Mapping):
            iterator = iteritems
        elif isinstance(obj, (Sequence, Set)) and not isinstance(obj, string_types):
            iterator = enumerate
        if iterator:
            if id(obj) not in memo:
                memo.add(id(obj))
                for path_component, value in iterator(obj):
                    for result in self.objwalk(value, path + (path_component,), memo):
                        yield result
                memo.remove(id(obj))
        else:
            yield path, obj
216981
def file_catalog(self):
        
        pathway = os.path.split(self.filename)
        self.catalog = sorted(os.listdir(pathway[0]))
755330
def get_paths(patterns, start_dirs=None, max_depth=1):
    """"""
    # Shortcut: if there is only one pattern, make sure we process just that.
    if len(patterns) == 1 and not start_dirs:
        pattern = patterns[0]
        directory = os.path.dirname(pattern)
        if directory:
            patterns = [os.path.basename(pattern)]
            start_dirs = directory
            max_depth = 1

    if not start_dirs or start_dirs == '.':
        start_dirs = os.getcwd()
    for start_dir in start_dirs.split(','):
        for root, dirs, files in os.walk(start_dir):  # pylint: disable=W0612
            if max_depth is not None:
                relpath = os.path.relpath(root, start=start_dir)
                depth = len(relpath.split(os.sep))
                if depth > max_depth:
                    continue
            names = []
            for pattern in patterns:
                names += fnmatch.filter(files, pattern)
            for name in names:
                path = os.path.join(root, name)
                yield path
653104
private RemoteIterator<LocatedFileStatus> methodBasedListPathWithLocation(
      final String src) throws IOException {
    if (namenodeProtocolProxy.isMethodSupported(
        "getLocatedPartialListing", String.class, byte[].class)) {
      return iteratorListing(src);
    } else {
      return arrayListing(src);
    }
  }
1208615
def _walk_all_files():
    
    # Init records
    record_dirs = set()
    result_set = set()

    for filename in Settings.paths:
        # Record dirs to put timestamps in later
        filename_full = os.path.abspath(filename)
        if Settings.recurse and os.path.isdir(filename_full):
            record_dirs.add(filename_full)

        walk_after = timestamp.get_walk_after(filename_full)
        results = walk_file(filename_full, walk_after, Settings.recurse)
        result_set = result_set.union(results)

    bytes_in = 0
    bytes_out = 0
    nag_about_gifs = False
    errors = []
    for result in result_set:
        res = result.get()
        if res.error:
            errors += [(res.final_filename, res.error)]
            continue
        bytes_in += res.bytes_in
        bytes_out += res.bytes_out
        nag_about_gifs = nag_about_gifs or res.nag_about_gifs

    return record_dirs, bytes_in, bytes_out, nag_about_gifs, errors
1086686
def file_path(self, depth=1, category=None, extension=None):
        
        file = self.file_name(category, extension)
        path = "/{0}".format(file)
        for _ in range(0, depth):
            path = "/{0}{1}".format(self.generator.word(), path)
        return path
734206
def get_files(path:PathOrStr, extensions:Collection[str]=None, recurse:bool=False,
              include:Optional[Collection[str]]=None)->FilePathList:
    ""
    if recurse:
        res = []
        for i,(p,d,f) in enumerate(os.walk(path)):
            # skip hidden dirs
            if include is not None and i==0:  d[:] = [o for o in d if o in include]
            else:                             d[:] = [o for o in d if not o.startswith('.')]
            res += _get_files(path, p, f, extensions)
        return res
    else:
        f = [o.name for o in os.scandir(path) if o.is_file()]
        return _get_files(path, path, f, extensions)
321284
def _GeneratorFromPath(path):
  """"""
  if not path:
    raise ValueError('path must be a valid string')
  if io_wrapper.IsTensorFlowEventsFile(path):
    return event_file_loader.EventFileLoader(path)
  else:
    return directory_watcher.DirectoryWatcher(
        path,
        event_file_loader.EventFileLoader,
        io_wrapper.IsTensorFlowEventsFile)
402841
def _EntriesGenerator(self):
    
    # Only the virtual root file has directory entries.
    volume_index = apfs_helper.APFSContainerPathSpecGetVolumeIndex(
        self.path_spec)
    if volume_index is not None:
      return

    location = getattr(self.path_spec, 'location', None)
    if location is None or location != self._file_system.LOCATION_ROOT:
      return

    fsapfs_container = self._file_system.GetAPFSContainer()

    for volume_index in range(0, fsapfs_container.number_of_volumes):
      yield apfs_container_path_spec.APFSContainerPathSpec(
          location='/apfs{0:d}'.format(volume_index + 1),
          volume_index=volume_index, parent=self.path_spec.parent)
75785
def get_filenodes_generator(self):
        
        for topnode, dirs, files in self.walk():
            for node in files:
                yield node