Modified version of https://docs.python.org/3/library/os.html#os.scandir
    that returns https://docs.python.org/3/library/os.html#os.DirEntry for files
    directly to take advantage of possible cached os.stat calls.
961764
def _walk(top):
    
    dirs = []
    nondirs = []

    try:
        scandir_iter = scandir(top)
    except OSError:
        return

    # Avoid repeated global lookups.
    get_next = next

    while True:
        try:
            entry = get_next(scandir_iter)
        except StopIteration:
            break
        except OSError:
            return

        try:
            is_dir = entry.is_dir()
        except OSError:
            is_dir = False

        if is_dir:
            dirs.append(entry)
        else:
            nondirs.append(entry)

    yield top, dirs, nondirs

    for dir_entry in dirs:
        for entry in walk(dir_entry.path):
            yield entry
1015338
def __get_datadir(self, app=False):
        
        # if 'datadir' in self.oseg.cache.data.keys():
        # if :
        #     directory = self.oseg.input_datapath_start
        if self.cache is not None:
            cache_loaddir = self.cache.get_or_none('loaddir')
            self.loaddir = str(cache_loaddir)
            # self.loaddir = str(self.cache.get_or_none('loaddir'))

        if self.loaddir is None:
            self.loaddir = ''

        directory = self.loaddir

        from PyQt4.QtGui import QFileDialog
        if not app:
            app_inner = QApplication(sys.argv)
        if self._skip_get_path_dialog_for_tests:
            dcmdir = self.datapath
        else:
            dcmdir = QFileDialog.getExistingDirectory(
                caption='Select DICOM Folder',
                options=QFileDialog.ShowDirsOnly,
                directory=directory
            )
            # pp.exec_()
        if not app:
            app_inner.exit(0)

        dcmdir = get_str(dcmdir)

        if len(dcmdir) > 0:


            # dcmdir = "%s" % (dcmdir)
            # dcmdir = dcmdir.encode("utf8")
            pass
        else:
            dcmdir = None

        if self.cache is not None:
            self.cache.update('loaddir', dcmdir)
        return str(dcmdir)
345854
def _get_last_cachefile_modification(self):
        

        try:
            mtime = os.path.getmtime(self._tld_list_path)
        except OSError:
            return None

        return datetime.fromtimestamp(mtime)
222874
def get_fileinfo(filespec, cache_dir=None):
    
    if cache_dir is None:
        cache_dir = tempfile.gettempdir()

    # Loads first science extension by default.
    # This prevents [None] to be loaded instead.
    idx = None
    name_ext = ''

    # User specified an index using bracket notation at end of path?
    match = re.match(r'^(.+)\[(.+)\]$', filespec)
    if match:
        filespec = match.group(1)
        idx = match.group(2)
        if ',' in idx:
            hduname, extver = idx.split(',')
            hduname = hduname.strip()
            if re.match(r'^\d+$', extver):
                extver = int(extver)
                idx = (hduname, extver)
                name_ext = "[%s,%d]" % idx
            else:
                idx = idx.strip()
                name_ext = "[%s]" % idx
        else:
            if re.match(r'^\d+$', idx):
                idx = int(idx)
                name_ext = "[%d]" % idx
            else:
                idx = idx.strip()
                name_ext = "[%s]" % idx
    else:
        filespec = filespec

    url = filespec
    filepath = None

    # Does this look like a URL?
    match = re.match(r"^(\w+)://(.+)$", filespec)
    if match:
        urlinfo = urllib.parse.urlparse(filespec)
        if urlinfo.scheme == 'file':
            # local file
            filepath = str(pathlib.Path(urlinfo.path))

        else:
            path, filename = os.path.split(urlinfo.path)
            filepath = get_download_path(url, filename, cache_dir)

    else:
        # Not a URL
        filepath = filespec
        url = pathlib.Path(os.path.abspath(filepath)).as_uri()

    ondisk = os.path.exists(filepath)

    dirname, fname = os.path.split(filepath)
    fname_pfx, fname_sfx = os.path.splitext(fname)
    name = fname_pfx + name_ext

    res = Bunch.Bunch(filepath=filepath, url=url, numhdu=idx,
                      name=name, idx=name_ext, ondisk=ondisk)
    return res
686198
function readDirAndStat (inputPath, cb) {

    if (typeof inputPath === "function") {
        cb = inputPath;
        inputPath = process.cwd();
    }

    fs.readdir(inputPath, (err, res) => {
        if (err) { return cb(err); }
        sameTime(bindy(res, (c, done) => {
            let fullPath = path.join(inputPath, c);
            fs.stat(fullPath, (err, stat) => {
                if (err) { return done(err); }
                done(null, {
                    path: fullPath
                  , stat: stat
                });
            });
        }), cb);
    });
}
1126118
def cache_dest(self, url, saltenv='base', cachedir=None):
        '''
        
        '''
        proto = urlparse(url).scheme

        if proto == '':
            # Local file path
            return url

        if proto == 'salt':
            url, senv = salt.utils.url.parse(url)
            if senv:
                saltenv = senv
            return salt.utils.path.join(
                self.opts['cachedir'],
                'files',
                saltenv,
                url.lstrip('|/'))

        return self._extrn_path(url, saltenv, cachedir=cachedir)
1043757
def get(self, prefix, url, schema_version=None):
        
        if not self.cache_dir:
            return None

        filename = self._get_cache_file(prefix, url)

        try:
            with open(filename, 'rb') as file:
                item = pickle.load(file)
            if schema_version and schema_version != item.schema:
                LOGGER.debug("Cache get %s %s: Wanted schema %d, got %d",
                             prefix, url,
                             schema_version, item.schema)
                return None
            return item
        except FileNotFoundError:
            pass
        except Exception:  # pylint:disable=broad-except
            _, msg, _ = sys.exc_info()
            LOGGER.warning("Cache get %s %s failed: %s", prefix, url, msg)

        return None
1056716
def getdir(self, source):
        """"""
        subdir = posixpath.join(self.pdir, source._tok)
        try:
            self.fs.rm(subdir, True)
        except Exception as e:
            logger.debug("Directory clear failed: %s" % e)
        self.fs.mkdirs(subdir)
        return subdir
122172
def cache_node(node, provider, opts):
    '''
    
    '''
    if isinstance(opts, dict):
        __opts__.update(opts)

    if 'update_cachedir' not in __opts__ or not __opts__['update_cachedir']:
        return

    if not os.path.exists(os.path.join(__opts__['cachedir'], 'active')):
        init_cachedir()

    base = os.path.join(__opts__['cachedir'], 'active')
    provider, driver = provider.split(':')
    prov_dir = os.path.join(base, driver, provider)
    if not os.path.exists(prov_dir):
        os.makedirs(prov_dir)
    path = os.path.join(prov_dir, '{0}.p'.format(node['name']))
    mode = 'wb' if six.PY3 else 'w'
    with salt.utils.files.fopen(path, mode) as fh_:
        salt.utils.msgpack.dump(node, fh_, encoding=MSGPACK_ENCODING)
1101691
def cache_dir(self, path, saltenv='base', include_empty=False,
                  include_pat=None, exclude_pat=None, cachedir=None):
        '''
        
        '''
        ret = []

        path = self._check_proto(salt.utils.data.decode(path))
        # We want to make sure files start with this *directory*, use
        # '/' explicitly because the master (that's generating the
        # list of files) only runs on POSIX
        if not path.endswith('/'):
            path = path + '/'

        log.info(
            'Caching directory \'%s\' for environment \'%s\'', path, saltenv
        )
        # go through the list of all files finding ones that are in
        # the target directory and caching them
        for fn_ in self.file_list(saltenv):
            fn_ = salt.utils.data.decode(fn_)
            if fn_.strip() and fn_.startswith(path):
                if salt.utils.stringutils.check_include_exclude(
                        fn_, include_pat, exclude_pat):
                    fn_ = self.cache_file(
                        salt.utils.url.create(fn_), saltenv, cachedir=cachedir)
                    if fn_:
                        ret.append(fn_)

        if include_empty:
            # Break up the path into a list containing the bottom-level
            # directory (the one being recursively copied) and the directories
            # preceding it
            # separated = string.rsplit(path, '/', 1)
            # if len(separated) != 2:
            #     # No slashes in path. (So all files in saltenv will be copied)
            #     prefix = ''
            # else:
            #     prefix = separated[0]
            cachedir = self.get_cachedir(cachedir)

            dest = salt.utils.path.join(cachedir, 'files', saltenv)
            for fn_ in self.file_list_emptydirs(saltenv):
                fn_ = salt.utils.data.decode(fn_)
                if fn_.startswith(path):
                    minion_dir = '{0}/{1}'.format(dest, fn_)
                    if not os.path.isdir(minion_dir):
                        os.makedirs(minion_dir)
                    ret.append(minion_dir)
        return ret
268668
def chdir(directory):
    
    try:
        olddir = os.getcwd()
    except OSError:
        olddir = None
    os.chdir(directory)
    return olddir
469165
def get_cached_files(url, server_name="", document_root=None):
    
    import glob
    url_info = process_url(url, server_name, document_root, check_security=False)

    # get path to cache directory with basename of file (no extension)
    filedir = os.path.dirname(url_info['requested_file'])
    fileglob = '{0}*{1}'.format(url_info['base_filename'], url_info['ext'])
    return glob.glob(os.path.join(filedir, fileglob))
774495
def get_file(self,
                 path,
                 dest='',
                 makedirs=False,
                 saltenv='base',
                 gzip=None,
                 cachedir=None):
        '''
        
        '''
        path, senv = salt.utils.url.split_env(path)
        if senv:
            saltenv = senv

        if not salt.utils.platform.is_windows():
            hash_server, stat_server = self.hash_and_stat_file(path, saltenv)
            try:
                mode_server = stat_server[0]
            except (IndexError, TypeError):
                mode_server = None
        else:
            hash_server = self.hash_file(path, saltenv)
            mode_server = None

        # Check if file exists on server, before creating files and
        # directories
        if hash_server == '':
            log.debug(
                'Could not find file \'%s\' in saltenv \'%s\'',
                path, saltenv
            )
            return False

        # If dest is a directory, rewrite dest with filename
        if dest is not None \
                and (os.path.isdir(dest) or dest.endswith(('/', '\\'))):
            dest = os.path.join(dest, os.path.basename(path))
            log.debug(
                'In saltenv \'%s\', \'%s\' is a directory. Changing dest to '
                '\'%s\'', saltenv, os.path.dirname(dest), dest
            )

        # Hash compare local copy with master and skip download
        # if no difference found.
        dest2check = dest
        if not dest2check:
            rel_path = self._check_proto(path)

            log.debug(
                'In saltenv \'%s\', looking at rel_path \'%s\' to resolve '
                '\'%s\'', saltenv, rel_path, path
            )
            with self._cache_loc(
                    rel_path, saltenv, cachedir=cachedir) as cache_dest:
                dest2check = cache_dest

        log.debug(
            'In saltenv \'%s\', ** considering ** path \'%s\' to resolve '
            '\'%s\'', saltenv, dest2check, path
        )

        if dest2check and os.path.isfile(dest2check):
            if not salt.utils.platform.is_windows():
                hash_local, stat_local = \
                    self.hash_and_stat_file(dest2check, saltenv)
                try:
                    mode_local = stat_local[0]
                except (IndexError, TypeError):
                    mode_local = None
            else:
                hash_local = self.hash_file(dest2check, saltenv)
                mode_local = None

            if hash_local == hash_server:
                return dest2check

        log.debug(
            'Fetching file from saltenv \'%s\', ** attempting ** \'%s\'',
            saltenv, path
        )
        d_tries = 0
        transport_tries = 0
        path = self._check_proto(path)
        load = {'path': path,
                'saltenv': saltenv,
                'cmd': '_serve_file'}
        if gzip:
            gzip = int(gzip)
            load['gzip'] = gzip

        fn_ = None
        if dest:
            destdir = os.path.dirname(dest)
            if not os.path.isdir(destdir):
                if makedirs:
                    try:
                        os.makedirs(destdir)
                    except OSError as exc:
                        if exc.errno != errno.EEXIST:  # ignore if it was there already
                            raise
                else:
                    return False
            # We need an open filehandle here, that's why we're not using a
            # with clause:
            fn_ = salt.utils.files.fopen(dest, 'wb+')  # pylint: disable=resource-leakage
        else:
            log.debug('No dest file found')

        while True:
            if not fn_:
                load['loc'] = 0
            else:
                load['loc'] = fn_.tell()
            data = self.channel.send(load, raw=True)
            if six.PY3:
                # Sometimes the source is local (eg when using
                # 'salt.fileserver.FSChan'), in which case the keys are
                # already strings. Sometimes the source is remote, in which
                # case the keys are bytes due to raw mode. Standardize on
                # strings for the top-level keys to simplify things.
                data = decode_dict_keys_to_str(data)
            try:
                if not data['data']:
                    if not fn_ and data['dest']:
                        # This is a 0 byte file on the master
                        with self._cache_loc(
                                data['dest'],
                                saltenv,
                                cachedir=cachedir) as cache_dest:
                            dest = cache_dest
                            with salt.utils.files.fopen(cache_dest, 'wb+') as ofile:
                                ofile.write(data['data'])
                    if 'hsum' in data and d_tries < 3:
                        # Master has prompted a file verification, if the
                        # verification fails, re-download the file. Try 3 times
                        d_tries += 1
                        hsum = salt.utils.hashutils.get_hash(dest, salt.utils.stringutils.to_str(data.get('hash_type', b'md5')))
                        if hsum != data['hsum']:
                            log.warning(
                                'Bad download of file %s, attempt %d of 3',
                                path, d_tries
                            )
                            continue
                    break
                if not fn_:
                    with self._cache_loc(
                            data['dest'],
                            saltenv,
                            cachedir=cachedir) as cache_dest:
                        dest = cache_dest
                        # If a directory was formerly cached at this path, then
                        # remove it to avoid a traceback trying to write the file
                        if os.path.isdir(dest):
                            salt.utils.files.rm_rf(dest)
                        fn_ = salt.utils.atomicfile.atomic_open(dest, 'wb+')
                if data.get('gzip', None):
                    data = salt.utils.gzip_util.uncompress(data['data'])
                else:
                    data = data['data']
                if six.PY3 and isinstance(data, str):
                    data = data.encode()
                fn_.write(data)
            except (TypeError, KeyError) as exc:
                try:
                    data_type = type(data).__name__
                except AttributeError:
                    # Shouldn't happen, but don't let this cause a traceback.
                    data_type = six.text_type(type(data))
                transport_tries += 1
                log.warning(
                    'Data transport is broken, got: %s, type: %s, '
                    'exception: %s, attempt %d of 3',
                    data, data_type, exc, transport_tries
                )
                self._refresh_channel()
                if transport_tries > 3:
                    log.error(
                        'Data transport is broken, got: %s, type: %s, '
                        'exception: %s, retry attempts exhausted',
                        data, data_type, exc
                    )
                    break

        if fn_:
            fn_.close()
            log.info(
                'Fetching file from saltenv \'%s\', ** done ** \'%s\'',
                saltenv, path
            )
        else:
            log.debug(
                'In saltenv \'%s\', we are ** missing ** the file \'%s\'',
                saltenv, path
            )

        return dest
763697
def archive(base_dir: str) -> int:
        

        LOGGER.debug('archive >>> base_dir: %s', base_dir)

        rv = int(time())
        timestamp_dir = join(base_dir, str(rv))
        makedirs(timestamp_dir, exist_ok=True)

        with SCHEMA_CACHE.lock:
            with open(join(timestamp_dir, 'schema'), 'w') as archive:
                print(json.dumps(SCHEMA_CACHE.schemata()), file=archive)

        with CRED_DEF_CACHE.lock:
            with open(join(timestamp_dir, 'cred_def'), 'w') as archive:
                print(json.dumps(CRED_DEF_CACHE), file=archive)

        with REVO_CACHE.lock:
            with open(join(timestamp_dir, 'revocation'), 'w') as archive:
                revo_cache_dict = {}
                for rr_id in REVO_CACHE:
                    revo_cache_dict[rr_id] = {
                        'rev_reg_def': REVO_CACHE[rr_id].rev_reg_def,
                        'rr_delta_frames': [vars(f) for f in REVO_CACHE[rr_id].rr_delta_frames],
                        'rr_state_frames': [vars(f) for f in REVO_CACHE[rr_id].rr_state_frames]
                    }
                print(json.dumps(revo_cache_dict), file=archive)

        LOGGER.debug('archive <<< %s', rv)
        return rv
733023
def cachedir_index_del(minion_id, base=None):
    '''
    
    '''
    base = init_cachedir(base)
    index_file = os.path.join(base, 'index.p')
    lock_file(index_file)

    if os.path.exists(index_file):
        mode = 'rb' if six.PY3 else 'r'
        with salt.utils.files.fopen(index_file, mode) as fh_:
            index = salt.utils.data.decode(
                salt.utils.msgpack.load(fh_, encoding=MSGPACK_ENCODING))
    else:
        return

    if minion_id in index:
        del index[minion_id]

    mode = 'wb' if six.PY3 else 'w'
    with salt.utils.files.fopen(index_file, mode) as fh_:
        salt.utils.msgpack.dump(index, fh_, encoding=MSGPACK_ENCODING)

    unlock_file(index_file)
587213
function crawlDirectoryWithFS(parent, sync) {
    return fsCmd('readdir', sync, parent).then(function (things) {
        return Q.all(things.map(function (file) {
            file = path.join(parent, file);
            return Q.all([file, fsCmd('stat', sync, file)]);
        }));
    }).then(function (stats) {
        return Q.all(stats.map(function (args) {
            var file = args[0], stat = args[1];
            if (stat.isDirectory()) {
                return crawlDirectoryWithFS(file, sync);
            }
            return file;
        }));
    }).then(function (grouped) {
        return [].concat.apply([], grouped);
    });
}
1174006
def get_pickled_ontology(filename):
    
    pickledfile = os.path.join(ONTOSPY_LOCAL_CACHE, filename + ".pickle")
    # pickledfile = ONTOSPY_LOCAL_CACHE + "/" + filename + ".pickle"
    if GLOBAL_DISABLE_CACHE:
        printDebug(
            "WARNING: DEMO MODE cache has been disabled in __init__.py ==============",
            "red")
    if os.path.isfile(pickledfile) and not GLOBAL_DISABLE_CACHE:
        try:
            return cPickle.load(open(pickledfile, "rb"))
        except:
            print(Style.DIM +
                  "** WARNING: Cache is out of date ** ...recreating it... " +
                  Style.RESET_ALL)
            return None
    else:
        return None