// Retry sending a message with exponential backoff; if over limit, bubble up the failed message.
495339
func (c *xmppGcmClient) retryMessage(cm CcsMessage, h MessageHandler) {
	c.messages.RLock()
	defer c.messages.RUnlock()
	if me, ok := c.messages.m[cm.MessageId]; ok {
		if me.backoff.sendAnother() {
			go func(m *messageLogEntry) {
				m.backoff.wait()
				c.send(*m.body)
			}(me)
		} else {
			debug("Exponential backoff failed over limit for message: ", me)
			go h(cm)
		}
	}
}
444629
func (b *ExponentialBackoff) Failure(event Event, err error) bool {
	atomic.AddUint64(&b.failures, 1)
	return false
}
218455
func (e *TokensController) retryOrForget(queue workqueue.RateLimitingInterface, key interface{}, requeue bool) {
	if !requeue {
		queue.Forget(key)
		return
	}

	requeueCount := queue.NumRequeues(key)
	if requeueCount < e.maxRetries {
		queue.AddRateLimited(key)
		return
	}

	klog.V(4).Infof("retried %d times: %#v", requeueCount, key)
	queue.Forget(key)
}
920830
func retryForIMDS(sender Sender, req *http.Request, maxAttempts int) (resp *http.Response, err error) {
	// copied from client.go due to circular dependency
	retries := []int{
		http.StatusRequestTimeout,      // 408
		http.StatusTooManyRequests,     // 429
		http.StatusInternalServerError, // 500
		http.StatusBadGateway,          // 502
		http.StatusServiceUnavailable,  // 503
		http.StatusGatewayTimeout,      // 504
	}
	// extra retry status codes specific to IMDS
	retries = append(retries,
		http.StatusNotFound,
		http.StatusGone,
		// all remaining 5xx
		http.StatusNotImplemented,
		http.StatusHTTPVersionNotSupported,
		http.StatusVariantAlsoNegotiates,
		http.StatusInsufficientStorage,
		http.StatusLoopDetected,
		http.StatusNotExtended,
		http.StatusNetworkAuthenticationRequired)

	// see https://docs.microsoft.com/en-us/azure/active-directory/managed-service-identity/how-to-use-vm-token#retry-guidance

	const maxDelay time.Duration = 60 * time.Second

	attempt := 0
	delay := time.Duration(0)

	for attempt < maxAttempts {
		resp, err = sender.Do(req)
		// retry on temporary network errors, e.g. transient network failures.
		// if we don't receive a response then assume we can't connect to the
		// endpoint so we're likely not running on an Azure VM so don't retry.
		if (err != nil && !isTemporaryNetworkError(err)) || resp == nil || resp.StatusCode == http.StatusOK || !containsInt(retries, resp.StatusCode) {
			return
		}

		// perform exponential backoff with a cap.
		// must increment attempt before calculating delay.
		attempt++
		// the base value of 2 is the "delta backoff" as specified in the guidance doc
		delay += (time.Duration(math.Pow(2, float64(attempt))) * time.Second)
		if delay > maxDelay {
			delay = maxDelay
		}

		select {
		case <-time.After(delay):
			// intentionally left blank
		case <-req.Context().Done():
			err = req.Context().Err()
			return
		}
	}
	return
}
713882
func Retry(ctx context.Context, p Policy, e Executer) error {
	b, cancel := p.Start(ctx)
	defer cancel()

	for Continue(b) {
		err := e.Execute(ctx)
		if err == nil {
			return nil
		}

		if IsPermanentError(err) {
			return errors.Wrap(err, `permanent error`)
		}
	}
	return errors.New(`retry attempts failed`)
}
315496
func backoffDuration(retryCount int) time.Duration {
	// Upper limit of delay at ~1 minute
	if retryCount > 7 {
		retryCount = 7
	}

	rand.Seed(time.Now().UnixNano())
	delay := (1 << uint(retryCount)) * (rand.Intn(150) + 500)
	return time.Duration(delay) * time.Millisecond
}
258791
func (bc Exponential) Backoff(retries int) time.Duration {
	if retries == 0 {
		return baseDelay
	}
	backoff, max := float64(baseDelay), float64(bc.MaxDelay)
	for backoff < max && retries > 0 {
		backoff *= factor
		retries--
	}
	if backoff > max {
		backoff = max
	}
	// Randomize backoff delays so that if a cluster of requests start at
	// the same time, they won't operate in lockstep.
	backoff *= 1 + jitter*(grpcrand.Float64()*2-1)
	if backoff < 0 {
		return 0
	}
	return time.Duration(backoff)
}
216118
func retry(ctx context.Context, f func() bool, delay time.Duration) error {
	if f() {
		return nil
	}

	ticker := time.NewTicker(delay)
	errChan := make(chan error)

	go func() {
		defer close(errChan)

		for {
			select {
			case <-ctx.Done():
				errChan <- ctx.Err()
				return
			case <-ticker.C:
				if f() {
					return
				}
			}
		}
	}()

	return <-errChan
}
626562
func (manager *Manager) retryingExpire(now time.Time) {
	manager.config.Logger.Tracef("[%s] expire looking for leases to expire\n", manager.logContext)
	defer manager.wg.Done()
	var err error
	for a := manager.startRetry(); a.Next(); {
		err = manager.expire(now)
		if !lease.IsTimeout(err) {
			break
		}
		if a.More() {
			manager.config.Logger.Tracef("[%s] timed out during expire, retrying...", manager.logContext)
		}
	}
	// Don't bother sending an error if we're dying - this avoids a
	// race in the tests.
	select {
	case <-manager.catacomb.Dying():
		manager.config.Logger.Tracef("[%s] expire exiting early do to manager shutdown", manager.logContext)
		return
	default:
	}

	if lease.IsTimeout(err) {
		// We don't crash on timeouts to avoid bouncing the API server.
		manager.config.Logger.Warningf("[%s] retrying timed out in expire", manager.logContext)
		return
	}
	if err != nil {
		manager.catacomb.Kill(errors.Trace(err))
	} else {
		// If we send back an error, then the main loop won't listen for expireDone
		select {
		case <-manager.catacomb.Dying():
			return
		case manager.expireDone <- struct{}{}:
		}
	}
}
1128453
func (rs *retryingSink) Write(events ...Event) error {
	rs.mu.Lock()
	defer rs.mu.Unlock()

retry:

	if rs.closed {
		return ErrSinkClosed
	}

	if !rs.proceed() {
		logrus.Warnf("%v encountered too many errors, backing off", rs.sink)
		rs.wait(rs.failures.backoff)
		goto retry
	}

	if err := rs.write(events...); err != nil {
		if err == ErrSinkClosed {
			// terminal!
			return err
		}

		logrus.Errorf("retryingsink: error writing events: %v, retrying", err)
		goto retry
	}

	return nil
}
1082696
func (rs *retryingSink) failure() {
	rs.failures.recent++
	rs.failures.last = time.Now().UTC()
}
789086
func retry(op Op, times int) Op {
	return func(req rets.Requester, urls rets.CapabilityURLs) error {
		var err error
		for ; times > 0; times-- {
			err = op(req, urls)
			if err == nil {
				return nil
			}
		}
		return err
	}
}
550744
func (f panicFailer) Fail(test Test, obtained, expected interface{}, msgs ...string) bool {
	obex := obexString(test, obtained, expected)
	panic(failString(test, obex, msgs...))
}
245774
func (f RetrierFunc) Retry(ctx context.Context, retry int, req *http.Request, resp *http.Response, err error) (time.Duration, bool, error) {
	return f(ctx, retry, req, resp, err)
}
496237
func (m *Message) RequeueWithoutBackoff(delay time.Duration) {
	m.doRequeue(delay, false)
}
143393
func Retry(ctx context.Context, f func() error) error {
	// container for atomic error value
	type errWrap struct {
		E error
	}

	// Try the function in a goroutine
	var errVal atomic.Value
	doneCh := make(chan struct{})
	go func() {
		defer close(doneCh)

		delay := time.Duration(0)
		for {
			// If our context ended, we want to exit right away.
			select {
			case <-ctx.Done():
				return
			case <-time.After(delay):
			}

			// Try the function call
			err := f()

			// return if we have no error, or a FatalError
			done := false
			switch e := err.(type) {
			case nil:
				done = true
			case Fatal:
				err = e.FatalError()
				done = true
			}

			errVal.Store(errWrap{err})

			if done {
				return
			}

			log.Printf("[WARN] retryable error: %v", err)

			delay *= 2

			if delay == 0 {
				delay = initialBackoffDelay
			}

			if delay > maxBackoffDelay {
				delay = maxBackoffDelay
			}

			log.Printf("[INFO] sleeping for %s", delay)
		}
	}()

	// Wait for completion
	select {
	case <-ctx.Done():
	case <-doneCh:
	}

	var lastErr error
	// Check if we got an error executing
	if ev, ok := errVal.Load().(errWrap); ok {
		lastErr = ev.E
	}

	// Check if we have a context error to check if we're interrupted or timeout
	switch ctx.Err() {
	case context.Canceled:
		return fmt.Errorf("interrupted - last error: %v", lastErr)
	case context.DeadlineExceeded:
		return fmt.Errorf("timeout - last error: %v", lastErr)
	}

	if lastErr != nil {
		return lastErr
	}
	return nil
}
28874
func (r *SyncResult) Fail(err error, msg string) {
	r.Error, r.Message = err, msg
}