// Duplicates the functionality of ExponentialBackoff but is specifically targeted towards ThriftCalls.
422752
func (r *realisClient) thriftCallWithRetries(thriftCall auroraThriftCall) (*aurora.Response, error) {
	var resp *aurora.Response
	var clientErr error
	var curStep int

	backoff := r.config.backoff
	duration := backoff.Duration

	for curStep = 0; curStep < backoff.Steps; curStep++ {

		// If this isn't our first try, backoff before the next try.
		if curStep != 0 {
			adjusted := duration
			if backoff.Jitter > 0.0 {
				adjusted = Jitter(duration, backoff.Jitter)
			}

			r.logger.Printf("A retriable error occurred during thrift call, backing off for %v before retry %v\n", adjusted, curStep)

			time.Sleep(adjusted)
			duration = time.Duration(float64(duration) * backoff.Factor)
		}

		// Only allow one go-routine make use or modify the thrift client connection.
		// Placing this in an anonymous function in order to create a new, short-lived stack allowing unlock
		// to be run in case of a panic inside of thriftCall.
		func() {
			r.lock.Lock()
			defer r.lock.Unlock()

			resp, clientErr = thriftCall()

			r.logger.TracePrintf("Aurora Thrift Call ended resp: %v clientErr: %v\n", resp, clientErr)
		}()

		// Check if our thrift call is returning an error. This is a retriable event as we don't know
		// if it was caused by network issues.
		if clientErr != nil {

			// Print out the error to the user
			r.logger.Printf("Client Error: %v\n", clientErr)

			// Determine if error is a temporary URL error by going up the stack
			e, ok := clientErr.(thrift.TTransportException)
			if ok {
				r.logger.DebugPrint("Encountered a transport exception")

				e, ok := e.Err().(*url.Error)
				if ok {
					// EOF error occurs when the server closes the read buffer of the client. This is common
					// when the server is overloaded and should be retried. All other errors that are permanent
					// will not be retried.
					if e.Err != io.EOF && !e.Temporary() {
						return nil, errors.Wrap(clientErr, "Permanent connection error")
					}
				}
			}

			// In the future, reestablish connection should be able to check if it is actually possible
			// to make a thrift call to Aurora. For now, a reconnect should always lead to a retry.
			r.ReestablishConn()

		} else {

			// If there was no client error, but the response is nil, something went wrong.
			// Ideally, we'll never encounter this but we're placing a safeguard here.
			if resp == nil {
				return nil, errors.New("Response from aurora is nil")
			}

			// Check Response Code from thrift and make a decision to continue retrying or not.
			switch responseCode := resp.GetResponseCode(); responseCode {

			// If the thrift call succeeded, stop retrying
			case aurora.ResponseCode_OK:
				return resp, nil

			// If the response code is transient, continue retrying
			case aurora.ResponseCode_ERROR_TRANSIENT:
				r.logger.Println("Aurora replied with Transient error code, retrying")
				continue

			// Failure scenarios, these indicate a bad payload or a bad config. Stop retrying.
			case aurora.ResponseCode_INVALID_REQUEST,
				aurora.ResponseCode_ERROR,
				aurora.ResponseCode_AUTH_FAILED,
				aurora.ResponseCode_JOB_UPDATING_ERROR:
				r.logger.Printf("Terminal Response Code %v from Aurora, won't retry\n", resp.GetResponseCode().String())
				return resp, errors.New(response.CombineMessage(resp))

				// The only case that should fall down to here is a WARNING response code.
				// It is currently not used as a response in the scheduler so it is unknown how to handle it.
			default:
				r.logger.DebugPrintf("unhandled response code %v received from Aurora\n", responseCode)
				return nil, errors.Errorf("unhandled response code from Aurora %v\n", responseCode.String())
			}
		}

	}

	r.logger.DebugPrintf("it took %v retries to complete this operation\n", curStep)

	if curStep > 1 {
		r.config.logger.Printf("retried this thrift call %d time(s)", curStep)
	}

	// Provide more information to the user wherever possible.
	if clientErr != nil {
		return nil, newRetryError(errors.Wrap(clientErr, "ran out of retries, including latest error"), curStep)
	} else {
		return nil, newRetryError(errors.New("ran out of retries"), curStep)
	}
}
109706
func RetryWithBackoff(backoff wait.Backoff, condition func() (bool, error)) (bool, error) {
	duration := backoff.Duration
	i := 0
	for {
		if i != 0 {
			adjusted := duration
			if backoff.Jitter > 0.0 {
				adjusted = wait.Jitter(duration, backoff.Jitter)
			}
			time.Sleep(adjusted)
			duration = time.Duration(float64(duration) * backoff.Factor)
		}

		i++

		done, err := condition()
		if done {
			return done, err
		}
		noMoreRetries := i >= backoff.Steps
		if !noMoreRetries && err != nil {
			glog.V(2).Infof("retrying after error %v", err)
		}

		if noMoreRetries {
			glog.Infof("hit maximum retries %d with error %v", i, err)
			return done, err
		}
	}
}
169854
func (s *DB) Callback() *Callback {
	s.parent.callbacks = s.parent.callbacks.clone()
	return s.parent.callbacks
}
261783
func WithBackoff(bf BackoffFunc) CallOption {
	return CallOption{applyFunc: func(o *options) {
		o.backoffFunc = BackoffFuncContext(func(ctx context.Context, attempt uint) time.Duration {
			return bf(attempt)
		})
	}}
}
398455
func (e *ExponentialStrategy) Backoff() backoff.Backoff {
	return &exponentialBackoff{
		first: e.opts.first,
		max:   e.opts.max.Nanoseconds(),
		rand:  e.opts.newRand(),
	}
}
816843
func GoSilentCall(title string, fx func(), cb func(interface{})) {
	go SilentRecoveryHandlerCallback(title, func() error {
		fx()
		return nil
	}, cb)
}
1146697
func WithExponentialBackoff(initialBackoff time.Duration, webhookFn func() error) error {
	backoff := wait.Backoff{
		Duration: initialBackoff,
		Factor:   1.5,
		Jitter:   0.2,
		Steps:    5,
	}

	var err error
	wait.ExponentialBackoff(backoff, func() (bool, error) {
		err = webhookFn()
		// these errors indicate a transient error that should be retried.
		if net.IsConnectionReset(err) || apierrors.IsInternalError(err) || apierrors.IsTimeout(err) || apierrors.IsTooManyRequests(err) {
			return false, nil
		}
		// if the error sends the Retry-After header, we respect it as an explicit confirmation we should retry.
		if _, shouldRetry := apierrors.SuggestsClientDelay(err); shouldRetry {
			return false, nil
		}
		if err != nil {
			return false, err
		}
		return true, nil
	})
	return err
}
355191
func computeNextBackoff(currBackoff time.Duration) time.Duration {
	// Double the current backoff, truncating if it exceeds our maximum.
	nextBackoff := 2 * currBackoff
	if nextBackoff > cfg.MaxBackoff {
		nextBackoff = cfg.MaxBackoff
	}

	// Using 1/10 of our duration as a margin, compute a random offset to
	// avoid the nodes entering connection cycles.
	margin := nextBackoff / 10

	var wiggle big.Int
	wiggle.SetUint64(uint64(margin))
	if _, err := rand.Int(rand.Reader, &wiggle); err != nil {
		// Randomizing is not mission critical, so we'll just return the
		// current backoff.
		return nextBackoff
	}

	// Otherwise add in our wiggle, but subtract out half of the margin so
	// that the backoff can tweaked by 1/20 in either direction.
	return nextBackoff + (time.Duration(wiggle.Uint64()) - margin/2)
}
1144386
func BackOff(attempts uint) int {
	upper := int(math.Min(float64(retryBackOffCap), float64(retryBackOffBase)*math.Pow(2.0, float64(attempts))))
	sleep := time.Duration(rand.Intn(upper)) * time.Millisecond
	time.Sleep(sleep)
	return int(sleep)
}
694504
func (r *serviceRegistry) callback(method string) *callback {
	elem := strings.SplitN(method, serviceMethodSeparator, 2)
	if len(elem) != 2 {
		return nil
	}
	r.mu.Lock()
	defer r.mu.Unlock()
	return r.services[elem[0]].callbacks[elem[1]]
}
705731
func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int {
	attemptCh := make(chan int)

	// computes the exponential backoff duration according to
	// https://www.awsarchitectureblog.com/2015/03/backoff.html
	exponentialBackoffWait := func(attempt int) time.Duration {
		// normalize jitter to the range [0, 1.0]
		if jitter < NoJitter {
			jitter = NoJitter
		}
		if jitter > MaxJitter {
			jitter = MaxJitter
		}

		//sleep = random_between(0, min(cap, base * 2 ** attempt))
		sleep := unit * time.Duration(1<<uint(attempt))
		if sleep > cap {
			sleep = cap
		}
		if jitter != NoJitter {
			sleep -= time.Duration(c.random.Float64() * float64(sleep) * jitter)
		}
		return sleep
	}

	go func() {
		defer close(attemptCh)
		for i := 0; i < maxRetry; i++ {
			select {
			// Attempts start from 1.
			case attemptCh <- i + 1:
			case <-doneCh:
				// Stop the routine.
				return
			}
			time.Sleep(exponentialBackoffWait(i))
		}
	}()
	return attemptCh
}
253267
func BackoffNanos(
	retry int,
	jitter bool,
	backoffFactor float64,
	initialBackoff time.Duration,
	maxBackoff time.Duration,
	rngFn RngFn,
) int64 {
	backoff := initialBackoff.Nanoseconds()
	if retry >= 1 {
		backoffFloat64 := float64(backoff) * math.Pow(backoffFactor, float64(retry-1))
		// math.Inf is also larger than math.MaxInt64.
		if backoffFloat64 > math.MaxInt64 {
			return maxBackoff.Nanoseconds()
		}
		backoff = int64(backoffFloat64)
	}
	// Validate the value of backoff to make sure Int63n() does not panic.
	if jitter && backoff >= 2 {
		half := backoff / 2
		backoff = half + rngFn(half)
	}
	if maxBackoff := maxBackoff.Nanoseconds(); backoff > maxBackoff {
		backoff = maxBackoff
	}
	return backoff
}
894975
func (b *Backoffer) Fork() (*Backoffer, context.CancelFunc) {
	ctx, cancel := context.WithCancel(b.ctx)
	return &Backoffer{
		ctx:        ctx,
		maxSleep:   b.maxSleep,
		totalSleep: b.totalSleep,
		errors:     b.errors,
		vars:       b.vars,
	}, cancel
}
1046395
func (c *OperatorRekeyCommand) backupRetrieve(client *api.Client) int {
	// Handle the different API requests
	var fn func() (*api.RekeyRetrieveResponse, error)
	switch strings.ToLower(strings.TrimSpace(c.flagTarget)) {
	case "barrier":
		fn = client.Sys().RekeyRetrieveBackup
	case "recovery", "hsm":
		fn = client.Sys().RekeyRetrieveRecoveryBackup
	default:
		c.UI.Error(fmt.Sprintf("Unknown target: %s", c.flagTarget))
		return 1
	}

	// Make the request
	storedKeys, err := fn()
	if err != nil {
		c.UI.Error(fmt.Sprintf("Error retrieving rekey stored keys: %s", err))
		return 2
	}

	secret := &api.Secret{
		Data: structs.New(storedKeys).Map(),
	}

	return OutputSecret(c.UI, secret)
}
62362
func MaxBackoff(t time.Duration) ExponentialOption {
	return func(options *exponentialOptions) {
		options.max = t
	}
}
32421
func (b *URLBackoff) CalculateBackoff(actualUrl *url.URL) time.Duration {
	return b.Backoff.Get(b.baseUrlKey(actualUrl))
}
1211218
func NewMockBackoffFrom(i backoff.Backoff) *MockBackoff {
	return &MockBackoff{
		CloneFunc: &BackoffCloneFunc{
			defaultHook: i.Clone,
		},
		NextIntervalFunc: &BackoffNextIntervalFunc{
			defaultHook: i.NextInterval,
		},
		ResetFunc: &BackoffResetFunc{
			defaultHook: i.Reset,
		},
	}
}