// Create an "in-memory GCS" that allows access to buckets of any name, each
// initially with empty contents. The supplied clock will be used for
// generating timestamps.
553378
func NewConn(clock timeutil.Clock) (c gcs.Conn) {
	typed := &conn{
		clock:   clock,
		buckets: make(map[string]gcs.Bucket),
	}

	typed.mu = syncutil.NewInvariantMutex(typed.checkInvariants)

	c = typed
	return
}
923562
func (ms *AggMetrics) GC() {
	for {
		unix := time.Duration(time.Now().UnixNano())
		diff := ms.gcInterval - (unix % ms.gcInterval)
		time.Sleep(diff + time.Minute)
		log.Info("checking for stale chunks that need persisting.")
		now := uint32(time.Now().Unix())
		chunkMinTs := now - uint32(ms.chunkMaxStale)
		metricMinTs := now - uint32(ms.metricMaxStale)

		// as this is the only goroutine that can delete from ms.Metrics
		// we only need to lock long enough to get the list of orgs, then for each org
		// get the list of active metrics.
		// It doesn't matter if new orgs or metrics are added while we iterate these lists.
		ms.RLock()
		orgs := make([]uint32, 0, len(ms.Metrics))
		for o := range ms.Metrics {
			orgs = append(orgs, o)
		}
		ms.RUnlock()
		for _, org := range orgs {
			orgActiveMetrics := promActiveMetrics.WithLabelValues(strconv.Itoa(int(org)))
			keys := make([]schema.Key, 0, len(ms.Metrics[org]))
			ms.RLock()
			for k := range ms.Metrics[org] {
				keys = append(keys, k)
			}
			ms.RUnlock()
			for _, key := range keys {
				gcMetric.Inc()
				ms.RLock()
				a := ms.Metrics[org][key]
				ms.RUnlock()
				points, stale := a.GC(now, chunkMinTs, metricMinTs)
				if stale {
					log.Debugf("metric %s is stale. Purging data from memory.", key)
					ms.Lock()
					delete(ms.Metrics[org], key)
					orgActiveMetrics.Set(float64(len(ms.Metrics[org])))
					// note: this is racey. if a metric has just become unstale, it may have created a new chunk,
					// pruning an older one. in which case we double-subtract those points
					// hard to fix and super rare. see https://github.com/grafana/metrictank/pull/1242
					totalPoints.DecUint64(uint64(points))
					ms.Unlock()
				}
			}
			ms.RLock()
			orgActive := len(ms.Metrics[org])
			orgActiveMetrics.Set(float64(orgActive))
			ms.RUnlock()

			// If this org has no keys, then delete the org from the map
			if orgActive == 0 {
				// To prevent races, we need to check that there are still no metrics for the org while holding a write lock
				ms.Lock()
				orgActive = len(ms.Metrics[org])
				if orgActive == 0 {
					delete(ms.Metrics, org)
				}
				ms.Unlock()
			}
		}

		// Get the totalActive across all orgs.
		totalActive := 0
		ms.RLock()
		for o := range ms.Metrics {
			totalActive += len(ms.Metrics[o])
		}
		ms.RUnlock()
		metricsActive.Set(totalActive)
	}
}
1060390
func CreateGCECloudWithCloud(config *CloudConfig, c cloud.Cloud) (*Cloud, error) {
	gceCloud, err := CreateGCECloud(config)
	if err == nil {
		gceCloud.c = c
	}
	return gceCloud, err
}
651072
func (storage *MessageBodyStorage) BucketArn() *gocf.StringExpr {
	return gocf.Join("",
		gocf.String("arn:aws:s3:::"),
		storage.bucketNameExpr)
}
82680
func (e *EncryptedRaftLogger) GC(index uint64, term uint64, keepOldSnapshots uint64) error {
	// Delete any older snapshots
	curSnapshot := fmt.Sprintf("%016x-%016x%s", term, index, ".snap")

	snapshots, err := ListSnapshots(e.snapDir())
	if err != nil {
		return err
	}

	// Ignore any snapshots that are older than the current snapshot.
	// Delete the others. Rather than doing lexical comparisons, we look
	// at what exists before/after the current snapshot in the slice.
	// This means that if the current snapshot doesn't appear in the
	// directory for some strange reason, we won't delete anything, which
	// is the safe behavior.
	curSnapshotIdx := -1
	var (
		removeErr      error
		oldestSnapshot string
	)

	for i, snapFile := range snapshots {
		if curSnapshotIdx >= 0 && i > curSnapshotIdx {
			if uint64(i-curSnapshotIdx) > keepOldSnapshots {
				err := os.Remove(filepath.Join(e.snapDir(), snapFile))
				if err != nil && removeErr == nil {
					removeErr = err
				}
				continue
			}
		} else if snapFile == curSnapshot {
			curSnapshotIdx = i
		}
		oldestSnapshot = snapFile
	}

	if removeErr != nil {
		return removeErr
	}

	// Remove any WAL files that only contain data from before the oldest
	// remaining snapshot.

	if oldestSnapshot == "" {
		return nil
	}

	// Parse index out of oldest snapshot's filename
	var snapTerm, snapIndex uint64
	_, err = fmt.Sscanf(oldestSnapshot, "%016x-%016x.snap", &snapTerm, &snapIndex)
	if err != nil {
		return errors.Wrapf(err, "malformed snapshot filename %s", oldestSnapshot)
	}

	wals, err := ListWALs(e.walDir())
	if err != nil {
		return err
	}

	found := false
	deleteUntil := -1

	for i, walName := range wals {
		var walSeq, walIndex uint64
		_, err = fmt.Sscanf(walName, "%016x-%016x.wal", &walSeq, &walIndex)
		if err != nil {
			return errors.Wrapf(err, "could not parse WAL name %s", walName)
		}

		if walIndex >= snapIndex {
			deleteUntil = i - 1
			found = true
			break
		}
	}

	// If all WAL files started with indices below the oldest snapshot's
	// index, we can delete all but the newest WAL file.
	if !found && len(wals) != 0 {
		deleteUntil = len(wals) - 1
	}

	for i := 0; i < deleteUntil; i++ {
		walPath := filepath.Join(e.walDir(), wals[i])
		l, err := fileutil.TryLockFile(walPath, os.O_WRONLY, fileutil.PrivateFileMode)
		if err != nil {
			return errors.Wrapf(err, "could not lock old WAL file %s for removal", wals[i])
		}
		err = os.Remove(walPath)
		l.Close()
		if err != nil {
			return errors.Wrapf(err, "error removing old WAL file %s", wals[i])
		}
	}

	return nil
}
1368604
func CreatePluginSysCCs(p *Provider) []SelfDescribingSysCC {
	var sdscs []SelfDescribingSysCC
	for _, pscc := range loadSysCCs(p) {
		sdscs = append(sdscs, &SysCCWrapper{SCC: pscc})
	}
	return sdscs
}
60948
func (t *Tx) createBucketIfNotExists(b []byte) (kv.Bucket, error) {
	if t.writable {
		bkt, ok := t.kv.buckets[string(b)]
		if !ok {
			bkt = &Bucket{btree.New(2)}
			t.kv.buckets[string(b)] = bkt
			return &bucket{
				Bucket:   bkt,
				writable: t.writable,
			}, nil
		}

		return &bucket{
			Bucket:   bkt,
			writable: t.writable,
		}, nil
	}

	return nil, kv.ErrTxNotWritable
}
236020
func (c *DNSCache) GC() []string {
	c.Lock()
	expiredEntries, _ := c.cleanupExpiredEntries(time.Now())
	overLimitEntries := c.cleanupOverLimitEntries()
	c.Unlock()
	return KeepUniqueNames(append(expiredEntries, overLimitEntries...))
}
116771
func GC(m *Map, filter *GCFilter) int {
	if filter.RemoveExpired {
		t, _ := bpf.GetMtime()
		tsec := t / 1000000000
		filter.Time = uint32(tsec)
	}

	return doGC(m, filter)
}
1074310
func main() {
	if len(os.Args) < 2 {
		fmt.Println("you must specify a bucket")
		return
	}

	sess := session.Must(session.NewSession())

	svc := s3.New(sess)

	i := 0
	err := svc.ListObjectsPages(&s3.ListObjectsInput{
		Bucket: &os.Args[1],
	}, func(p *s3.ListObjectsOutput, last bool) (shouldContinue bool) {
		fmt.Println("Page,", i)
		i++

		for _, obj := range p.Contents {
			fmt.Println("Object:", *obj.Key)
		}
		return true
	})
	if err != nil {
		fmt.Println("failed to list objects", err)
		return
	}
}
862612
func NewGCSService(filename string) (*GCSService, error) {
	ctx := context.Background()
	client, err := storage.NewClient(ctx, option.WithServiceAccountFile(filename))
	if err != nil {
		return nil, err
	}

	service := &GCSService{
		Client: client,
	}

	return service, nil
}
1064214
func (e *ShowExec) bucketsToRows(dbName, tblName, partitionName, colName string, numOfCols int, hist statistics.Histogram) error {
	isIndex := 0
	if numOfCols > 0 {
		isIndex = 1
	}
	for i := 0; i < hist.Len(); i++ {
		lowerBoundStr, err := statistics.ValueToString(hist.GetLower(i), numOfCols)
		if err != nil {
			return errors.Trace(err)
		}
		upperBoundStr, err := statistics.ValueToString(hist.GetUpper(i), numOfCols)
		if err != nil {
			return errors.Trace(err)
		}
		e.appendRow([]interface{}{
			dbName,
			tblName,
			partitionName,
			colName,
			isIndex,
			i,
			hist.Buckets[i].Count,
			hist.Buckets[i].Repeat,
			lowerBoundStr,
			upperBoundStr,
		})
	}
	return nil
}
491276
func (g Generationer) StorageGeneration() (initTime time.Time, random string, err error) {
	f, err := os.Open(g.generationFile())
	if os.IsNotExist(err) {
		if err = g.ResetStorageGeneration(); err != nil {
			return
		}
		f, err = os.Open(g.generationFile())
	}
	if err != nil {
		return
	}
	defer f.Close()
	bs, err := ioutil.ReadAll(f)
	if err != nil {
		return
	}
	if i := bytes.IndexByte(bs, '\n'); i != -1 {
		bs = bs[:i]
	}
	if fi, err := f.Stat(); err == nil {
		initTime = fi.ModTime()
	}
	random = string(bs)
	return
}
757357
func (c *APIStore) CreateBucket(name string) error {
	bucket := &storage.Bucket{Name: name}
	_, err := c.service.Buckets.Insert(c.project, bucket).Do()
	return err
}
797523
func (h *Head) gc() {
	// Only data strictly lower than this timestamp must be deleted.
	mint := h.MinTime()

	// Drop old chunks and remember series IDs and hashes if they can be
	// deleted entirely.
	deleted, chunksRemoved := h.series.gc(mint)
	seriesRemoved := len(deleted)

	h.metrics.seriesRemoved.Add(float64(seriesRemoved))
	h.metrics.series.Sub(float64(seriesRemoved))
	h.metrics.chunksRemoved.Add(float64(chunksRemoved))
	h.metrics.chunks.Sub(float64(chunksRemoved))

	// Remove deleted series IDs from the postings lists.
	h.postings.Delete(deleted)

	if h.wal != nil {
		_, last, _ := h.wal.Segments()
		h.deletedMtx.Lock()
		// Keep series records until we're past segment 'last'
		// because the WAL will still have samples records with
		// this ref ID. If we didn't keep these series records then
		// on start up when we replay the WAL, or any other code
		// that reads the WAL, wouldn't be able to use those
		// samples since we would have no labels for that ref ID.
		for ref := range deleted {
			h.deleted[ref] = last
		}
		h.deletedMtx.Unlock()
	}

	// Rebuild symbols and label value indices from what is left in the postings terms.
	symbols := make(map[string]struct{}, len(h.symbols))
	values := make(map[string]stringset, len(h.values))

	if err := h.postings.Iter(func(t labels.Label, _ index.Postings) error {
		symbols[t.Name] = struct{}{}
		symbols[t.Value] = struct{}{}

		ss, ok := values[t.Name]
		if !ok {
			ss = stringset{}
			values[t.Name] = ss
		}
		ss.set(t.Value)
		return nil
	}); err != nil {
		// This should never happen, as the iteration function only returns nil.
		panic(err)
	}

	h.symMtx.Lock()

	h.symbols = symbols
	h.values = values

	h.symMtx.Unlock()
}
609485
func (agg *Aggregator) GC(now, chunkMinTs, metricMinTs, lastWriteTime uint32) (uint32, bool) {
	var points uint32
	stale := true

	if lastWriteTime+agg.span > chunkMinTs {
		// Last datapoint was less than one aggregation window before chunkMinTs, hold out for more data
		return 0, false
	}

	// Haven't seen datapoints in an entire aggregation window before chunkMinTs, time to flush
	if agg.agg.Cnt != 0 {
		agg.flush()
	}

	if agg.minMetric != nil {
		p, s := agg.minMetric.GC(now, chunkMinTs, metricMinTs)
		stale = stale && s
		points += p

	}
	if agg.maxMetric != nil {
		p, s := agg.maxMetric.GC(now, chunkMinTs, metricMinTs)
		stale = stale && s
		points += p
	}
	if agg.sumMetric != nil {
		p, s := agg.sumMetric.GC(now, chunkMinTs, metricMinTs)
		stale = stale && s
		points += p
	}
	if agg.cntMetric != nil {
		p, s := agg.cntMetric.GC(now, chunkMinTs, metricMinTs)
		stale = stale && s
		points += p
	}
	if agg.lstMetric != nil {
		p, s := agg.lstMetric.GC(now, chunkMinTs, metricMinTs)
		stale = stale && s
		points += p
	}

	return points, stale
}
454941
func (l *gcsGateway) CleanupGCSMinioSysTmp(ctx context.Context) {
	for {
		it := l.client.Buckets(ctx, l.projectID)
		for {
			attrs, err := it.Next()
			if err != nil {
				if err != iterator.Done {
					ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{})
					logger.LogIf(ctx, err)
				}
				break
			}
			l.CleanupGCSMinioSysTmpBucket(ctx, attrs.Name)
		}
		// Run the cleanup loop every 1 day.
		time.Sleep(gcsCleanupInterval)
	}
}