Process a collection of buckets.

    For each bucket fetch location, versioning and size and
    then kickoff processing strategy based on size.
546105
def process_bucket_set(account_info, buckets):
    
    region_clients = {}
    log = logging.getLogger('salactus.bucket-set')
    log.info("processing account %s", account_info)
    session = get_session(account_info)
    client = session.client('s3', config=s3config)

    for b in buckets:
        bid = bucket_id(account_info, b)
        with bucket_ops(bid):
            info = {'name': b}
            error = None

            try:
                location = client.get_bucket_location(
                    Bucket=b).get('LocationConstraint')
            except Exception as e:
                error = e
                location = None

            if location is None:
                region = "us-east-1"
            elif location == 'EU':
                region = "eu-west-1"
            else:
                region = location

            if (account_info.get('regions', ()) and
                    region not in account_info.get('regions', ())):
                continue

            info['region'] = region
            if region not in region_clients:
                region_clients.setdefault(region, {})
                region_clients[region]['s3'] = s3 = session.client(
                    's3', region_name=region, config=s3config)
                region_clients[region]['cloudwatch'] = cw = session.client(
                    'cloudwatch', region_name=region, config=s3config)
            else:
                s3 = region_clients[region]['s3']
                cw = region_clients[region]['cloudwatch']

            try:
                info['keycount'] = bucket_key_count(cw, info)
            except Exception:
                raise
            else:
                connection.hset('bucket-sizes', bid, info['keycount'])

            if error:
                raise error

            connection.hset('bucket-regions', bid, region)

            versioning = s3.get_bucket_versioning(Bucket=b)
            info['versioned'] = (
                versioning and versioning.get('Status', '')
                in ('Enabled', 'Suspended') or False)
            connection.hset('bucket-versions', bid, int(info['versioned']))

            log.info("processing bucket %s", info)
            connection.hset('bucket-starts', bid, time.time())
            dispatch_object_source(s3, account_info, bid, info)
1195024
private void prefetch(long storageIndex) {
        long startIndex = storageIndex / BYTES_IN_DATA;
        // Inc to next bucket
        startIndex += 128;

        if (mPrefetchedBuckets.contains(startIndex)) {
            mPrefetchedBuckets.remove(startIndex);
            return;
        }

        for (int i = 0; i < BUCKETS_TO_PREFETCH; i++) {
            if ((startIndex + i) > (mNodeNumbers / 128)) {
                return;
            }

            mRtx.moveTo(startIndex);
            startIndex += 128;
        }
    }
901512
private void readObject (ObjectInputStream s)
         throws IOException, ClassNotFoundException
    {
        // read in number of buckets and allocate the bucket array
        _buckets = new Record[s.readInt()];
        _loadFactor = s.readFloat();

        // read in size (number of mappings)
        int size = s.readInt();

        // read the keys and values
        for (int i=0; i<size; i++) {
            put(s.readInt(), s.readInt());
        }
    }
1153524
def get_files(client, bucket, prefix=''):
    """"""
    bucket = client.get_bucket(bucket)
    files = list(bucket.list_blobs(prefix=prefix))    
    return files
1006659
def _init(creds, bucket, multiple_env, environment, prefix, s3_cache_expire):
    '''
    
    '''

    cache_file = _get_buckets_cache_filename(bucket, prefix)
    exp = time.time() - s3_cache_expire

    # check if cache_file exists and its mtime
    if os.path.isfile(cache_file):
        cache_file_mtime = os.path.getmtime(cache_file)
    else:
        # file does not exists then set mtime to 0 (aka epoch)
        cache_file_mtime = 0

    expired = (cache_file_mtime <= exp)

    log.debug(
        'S3 bucket cache file %s is %sexpired, mtime_diff=%ss, expiration=%ss',
        cache_file,
        '' if expired else 'not ',
        cache_file_mtime - exp,
        s3_cache_expire
    )

    if expired:
        pillars = _refresh_buckets_cache_file(creds, cache_file, multiple_env,
                                           environment, prefix)
    else:
        pillars = _read_buckets_cache_file(cache_file)

    log.debug('S3 bucket retrieved pillars %s', pillars)
    return pillars
620395
def process_update_records(update_records):
    """"""
    events = sorted(update_records, key=lambda x: x['account'])

    # Group records by account for more efficient processing
    for account_id, events in groupby(events, lambda x: x['account']):
        events = list(events)

        # Grab the bucket names (de-dupe events):
        buckets = {}
        for event in events:
            # If the creation date is present, then use it:
            bucket_event = buckets.get(event['detail']['requestParameters']['bucketName'], {
                'creationDate': event['detail']['requestParameters'].get('creationDate')
            })
            bucket_event.update(event['detail']['requestParameters'])

            buckets[event['detail']['requestParameters']['bucketName']] = bucket_event
            buckets[event['detail']['requestParameters']['bucketName']]['eventDetails'] = event

        # Query AWS for current configuration
        for b_name, item in buckets.items():
            LOG.debug(f'[~] Processing Create/Update for: {b_name}')
            # If the bucket does not exist, then simply drop the request --
            # If this happens, there is likely a Delete event that has occurred and will be processed soon.
            try:
                bucket_details = get_bucket(b_name,
                                            account_number=account_id,
                                            include_created=(item.get('creationDate') is None),
                                            assume_role=HISTORICAL_ROLE,
                                            region=CURRENT_REGION)
                if bucket_details.get('Error'):
                    LOG.error(f"[X] Unable to fetch details about bucket: {b_name}. "
                              f"The error details are: {bucket_details['Error']}")
                    continue

            except ClientError as cerr:
                if cerr.response['Error']['Code'] == 'NoSuchBucket':
                    LOG.warning(f'[?] Received update request for bucket: {b_name} that does not '
                                'currently exist. Skipping.')
                    continue

                # Catch Access Denied exceptions as well:
                if cerr.response['Error']['Code'] == 'AccessDenied':
                    LOG.error(f'[X] Unable to fetch details for S3 Bucket: {b_name} in {account_id}. Access is Denied. '
                              'Skipping...')
                    continue
                raise Exception(cerr)

            # Pull out the fields we want:
            data = {
                'arn': f'arn:aws:s3:::{b_name}',
                'principalId': cloudwatch.get_principal(item['eventDetails']),
                'userIdentity': cloudwatch.get_user_identity(item['eventDetails']),
                'userAgent': item['eventDetails']['detail'].get('userAgent'),
                'sourceIpAddress': item['eventDetails']['detail'].get('sourceIPAddress'),
                'requestParameters': item['eventDetails']['detail'].get('requestParameters'),
                'accountId': account_id,
                'eventTime': item['eventDetails']['detail']['eventTime'],
                'BucketName': b_name,
                'Region': bucket_details.pop('Region'),
                # Duplicated in top level and configuration for secondary index
                'Tags': bucket_details.pop('Tags', {}) or {},
                'eventSource': item['eventDetails']['detail']['eventSource'],
                'eventName': item['eventDetails']['detail']['eventName'],
                'version': VERSION
            }

            # Remove the fields we don't care about:
            del bucket_details['Arn']
            del bucket_details['GrantReferences']
            del bucket_details['_version']
            del bucket_details['Name']

            if not bucket_details.get('CreationDate'):
                bucket_details['CreationDate'] = item['creationDate']

            data['configuration'] = bucket_details

            current_revision = CurrentS3Model(**data)
            current_revision.save()
1011065
def get_buckets(self, bucket_type=None, timeout=None):
        
        bucket_type = self._get_bucket_type(bucket_type)
        url = self.bucket_list_path(bucket_type=bucket_type,
                                    timeout=timeout)
        status, headers, body = self._request('GET', url)

        if status == 200:
            props = json.loads(bytes_to_str(body))
            return props['buckets']
        else:
            raise RiakError('Error getting buckets.')
905372
function processFile (file, version, versions, targetVersions) {
  processTags(file.content, version, versions, file)
  processVersioned(file.content, version, versions)
  processVersionLists(file.content, version, targetVersions)
  return file
}
325622
def list_buckets(self):
        
        details = self._details(
            method=b"GET",
            url_context=self._url_context(),
        )
        query = self._query_factory(details)
        d = self._submit(query)
        d.addCallback(self._parse_list_buckets)
        return d
1362322
def get_bucket(self, hash_name, bucket_key):
        
        if hash_name in self.buckets:
            if bucket_key in self.buckets[hash_name]:
                return self.buckets[hash_name][bucket_key]
        return []
385205
def get_buckets(min_length, max_length, bucket_count):
    '''
    
    '''
    if bucket_count <= 0:
        return [max_length]
    unit_length = int((max_length - min_length) // (bucket_count))
    buckets = [min_length + unit_length *
               (i + 1) for i in range(0, bucket_count)]
    buckets[-1] = max_length
    return buckets
480430
def bucket_lister(bucket, prefix='', delimiter='', marker='', headers=None):
    
    more_results = True
    k = None
    while more_results:
        rs = bucket.get_all_keys(prefix=prefix, marker=marker,
                                 delimiter=delimiter, headers=headers)
        for k in rs:
            yield k
        if k:
            marker = rs.next_marker or k.name
        more_results= rs.is_truncated
744393
def process(self):
    
    hunt = self.grr_api.Hunt(self.hunt_id).Get()
    self.state.output = self.collect_hunt_results(hunt)
243181
protected void loadAndReplace(K key) {
    Entry e;
    for (;;) {
      e = lookupOrNewEntry(key);
      synchronized (e) {
        e.waitForProcessing();
        if (e.isGone()) {
          metrics.goneSpin();
          continue;
        }
        e.startProcessing(Entry.ProcessingState.LOAD);
        break;
      }
    }
    boolean _finished = false;
    try {
      load(e);
      _finished = true;
    } finally {
      e.ensureAbort(_finished);
    }
  }
183251
def _get(self, name, interval, config, timestamp, **kws):
    '''
    
    '''
    i_bucket, r_bucket, i_key, r_key = self._calc_keys(config, name, timestamp)
    fetch = kws.get('fetch') or self._type_get
    process_row = kws.get('process_row') or self._process_row

    rval = OrderedDict()
    if config['coarse']:
      data = process_row( fetch(self._client, i_key) )
      rval[ config['i_calc'].from_bucket(i_bucket) ] = data
    else:
      # First fetch all of the resolution buckets for this set.
      resolution_buckets = sorted(map(int,self._client.smembers(i_key)))

      # Create a pipe and go fetch all the data for each.
      # TODO: turn off transactions here?
      pipe = self._client.pipeline(transaction=False)
      for bucket in resolution_buckets:
        r_key = '%s:%s'%(i_key, bucket)   # TODO: make this the "resolution_bucket" closure?
        fetch(pipe, r_key)
      res = pipe.execute()

      for idx,data in enumerate(res):
        data = process_row(data)
        rval[ config['r_calc'].from_bucket(resolution_buckets[idx]) ] = data

    return rval
479611
public void init( List<T> list , int bucketSize ) {
		if( list.size() < bucketSize ) {
			throw new RuntimeException("There needs to be more than or equal to elements in the 'list' that there are in the bucket");
		}

		this.k = bucketSize;
		this.c = bucketSize - 1;
		N = list.size();

		bins = new int[ bucketSize ];

		for( int i = 0; i < bins.length; i++ ) {
			bins[i] = i;
		}

		this.a = list;
		state = 1;
	}
705564
private void loadBuckets() {
        Properties props = new Properties();
        try (FileReader reader = new FileReader(savedRates)) {
            props.load(reader);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        for (Entry<Object, Object> entry : props.entrySet()) {
            String key = entry.getKey().toString();
            String value = entry.getValue().toString();
            String [] split = value.split("="); //$NON-NLS-1$
            long count = new Long(split[0]);
            long last = new Long(split[1]);
            RateLimiterBucket bucket = new RateLimiterBucket();
            bucket.setCount(count);
            bucket.setLast(last);
            this.buckets.put(key, bucket);
        }
    }