Return the contents of the buckets cache file
499569
def _read_buckets_cache_file(cache_file):
    '''
    
    '''

    log.debug('Reading buckets cache file')

    with salt.utils.files.fopen(cache_file, 'rb') as fp_:
        data = pickle.load(fp_)

    return data
1205613
def _storage_list_keys(bucket, pattern):
  
  data = [{'Name': item.metadata.name,
           'Type': item.metadata.content_type,
           'Size': item.metadata.size,
           'Updated': item.metadata.updated_on}
          for item in _storage_get_keys(bucket, pattern)]
  return datalab.utils.commands.render_dictionary(data, ['Name', 'Type', 'Size', 'Updated'])
1011065
def get_buckets(self, bucket_type=None, timeout=None):
        
        bucket_type = self._get_bucket_type(bucket_type)
        url = self.bucket_list_path(bucket_type=bucket_type,
                                    timeout=timeout)
        status, headers, body = self._request('GET', url)

        if status == 200:
            props = json.loads(bytes_to_str(body))
            return props['buckets']
        else:
            raise RiakError('Error getting buckets.')
1046187
def get_cached(self, path, cache_name, **kwargs):
        
        if gw2api.cache_dir and gw2api.cache_time and cache_name:
            cache_file = os.path.join(gw2api.cache_dir, cache_name)
            if mtime(cache_file) >= time.time() - gw2api.cache_time:
                with open(cache_file, "r") as fp:
                    tmp = json.load(fp)
                return self.make_response(tmp["data"], tmp["meta"])
        else:
            cache_file = None

        meta, data = self._get(path, **kwargs)

        if cache_file:
            with open(cache_file, "w") as fp:
                json.dump({"meta": meta, "data": data}, fp, indent=2)

        return self.make_response(data, meta)
1310784
def get_from_cache(url: str, cache_dir: str = None) -> str:
    
    if cache_dir is None:
        cache_dir = CACHE_DIRECTORY

    os.makedirs(cache_dir, exist_ok=True)

    # Get eTag to add to filename, if it exists.
    if url.startswith("s3://"):
        etag = s3_etag(url)
    else:
        response = requests.head(url, allow_redirects=True)
        if response.status_code != 200:
            raise IOError("HEAD request failed for url {} with status code {}"
                          .format(url, response.status_code))
        etag = response.headers.get("ETag")

    filename = url_to_filename(url, etag)

    # get cache path to put the file
    cache_path = os.path.join(cache_dir, filename)

    if not os.path.exists(cache_path):
        # Download to temporary file, then copy to cache dir once finished.
        # Otherwise you get corrupt cache entries if the download gets interrupted.
        with tempfile.NamedTemporaryFile() as temp_file:
            logger.info("%s not found in cache, downloading to %s", url, temp_file.name)

            # GET file object
            if url.startswith("s3://"):
                s3_get(url, temp_file)
            else:
                http_get(url, temp_file)

            # we are copying the file before closing it, so flush to avoid truncation
            temp_file.flush()
            # shutil.copyfileobj() starts at the current position, so go to the start
            temp_file.seek(0)

            logger.info("copying %s to cache at %s", temp_file.name, cache_path)
            with open(cache_path, 'wb') as cache_file:
                shutil.copyfileobj(temp_file, cache_file)

            logger.info("creating metadata file for %s", cache_path)
            meta = {'url': url, 'etag': etag}
            meta_path = cache_path + '.json'
            with open(meta_path, 'w') as meta_file:
                json.dump(meta, meta_file)

            logger.info("removing temp file %s", temp_file.name)

    return cache_path
289105
def retrieve(self, key):
        """"""
        column_file = os.path.join(self._hash_dir, '%s.json' % key)
        cache_file = os.path.join(self._hash_dir, '%s.npy' % key)

        if os.path.exists(cache_file):
            data = np.load(cache_file)
            if os.path.exists(column_file):
                with open(column_file, 'r') as json_file:
                    columns = json.load(json_file)
                data = pd.DataFrame(data, columns=columns)
        else:
            return None

        return data
665018
def get_contents(diff_part):
    
    old_sha = get_old_sha(diff_part)
    old_filename = get_old_filename(diff_part)
    old_contents = get_old_contents(old_sha, old_filename)
    new_filename = get_new_filename(diff_part)
    new_contents = get_new_contents(new_filename)
    return old_contents, new_contents
292653
def cache_info(cache_dir, cache_key):
    
    filename = os.path.join(cache_dir, cache_key)
    return os.path.getmtime(filename) if os.path.exists(filename) else 0
177838
def glob(self, filename):
        """"""
        # Only support prefix with * at the end and no ? in the string
        star_i = filename.find('*')
        quest_i = filename.find('?')
        if quest_i >= 0:
            raise NotImplementedError(
                "{} not supported by compat glob".format(filename))
        if star_i != len(filename) - 1:
            # Just return empty so we can use glob from directory watcher
            #
            # TODO: Remove and instead handle in GetLogdirSubdirectories.
            # However, we would need to handle it for all non-local registered
            # filesystems in some way.
            return []
        filename = filename[:-1]
        client = boto3.client("s3")
        bucket, path = self.bucket_and_path(filename)
        p = client.get_paginator("list_objects")
        keys = []
        for r in p.paginate(Bucket=bucket, Prefix=path):
            for o in r.get("Contents", []):
                key = o["Key"][len(path):]
                if key:  # Skip the base dir, which would add an empty string
                    keys.append(filename + key)
        return keys
830479
def _get_contents(self):
        """"""
        def files():
            for value in super(GlobBundle, self)._get_contents():
                for path in glob.glob(value):
                    yield path
        return list(files())
246344
def get(self, bucket: str, key: str) -> bytes:
        
        bucket_obj = self._ensure_bucket_loaded(bucket)
        blob_obj = bucket_obj.blob(key)

        try:
            return blob_obj.download_as_string()
        except NotFound:
            raise BlobNotFoundError(f"Could not find gs://{bucket}/{key}")
418208
def cache_master(self, saltenv='base', cachedir=None):
        '''
        
        '''
        ret = []
        for path in self.file_list(saltenv):
            ret.append(
                self.cache_file(
                    salt.utils.url.create(path), saltenv, cachedir=cachedir)
            )
        return ret
879487
def _init():
    '''
    
    '''
    cache_file = _get_buckets_cache_filename()
    exp = time.time() - S3_CACHE_EXPIRE

    # check mtime of the buckets files cache
    metadata = None
    try:
        if os.path.getmtime(cache_file) > exp:
            metadata = _read_buckets_cache_file(cache_file)
    except OSError:
        pass

    if metadata is None:
        # bucket files cache expired or does not exist
        metadata = _refresh_buckets_cache_file(cache_file)

    return metadata
1000397
def gather_cache(self):
        '''
        
        '''
        cache = {'grains': {}, 'pillar': {}}
        if self.grains or self.pillar:
            if self.opts.get('minion_data_cache'):
                minions = self.cache.list('minions')
                if not minions:
                    return cache
                for minion in minions:
                    total = self.cache.fetch('minions/{0}'.format(minion), 'data')

                    if 'pillar' in total:
                        if self.pillar_keys:
                            for key in self.pillar_keys:
                                if key in total['pillar']:
                                    cache['pillar'][minion][key] = total['pillar'][key]
                        else:
                            cache['pillar'][minion] = total['pillar']
                    else:
                        cache['pillar'][minion] = {}

                    if 'grains' in total:
                        if self.grain_keys:
                            for key in self.grain_keys:
                                if key in total['grains']:
                                    cache['grains'][minion][key] = total['grains'][key]
                        else:
                            cache['grains'][minion] = total['grains']
                    else:
                        cache['grains'][minion] = {}
        return cache
16783
def list(self, path, filename=None, start=None, stop=None, recursive=False, directories=False):
        
        storageScheme, keys = self.getkeys(
            path, filename=filename, directories=directories, recursive=recursive)
        keys = [storageScheme + ":///" + key.bucket.name + "/" + key.name for key in keys]
        keys.sort()
        keys = select(keys, start, stop)
        return keys
549401
def [](key)
      return @cache[key] if @cache.key?(key)

      path = path_to(hash(key))
      if disk_cache_enabled? && File.file?(path) && File.readable?(path)
        @cache[key] = load(path)
      else
        raise
      end
    end
361426
def package_contents(self):
        ''

        if self.contents_cache:
            return self.contents_cache

        # Get a list of ZipInfo objects.
        files = self.zf.infolist()
        out_files = {}

        # Iterate through each file in the XPI.
        for file_ in files:

            file_doc = {'name': file_.filename,
                        'size': file_.file_size,
                        'name_lower': file_.filename.lower()}

            file_doc['extension'] = file_doc['name_lower'].split('.')[-1]

            out_files[file_.filename] = file_doc

        self.contents_cache = out_files
        return out_files