s3 cache client-side support

Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
This commit is contained in:
CrazyMax
2022-08-30 17:00:10 +02:00
parent 6804bcbf12
commit 57d22a7bd1
305 changed files with 45257 additions and 6 deletions

View File

@@ -0,0 +1,156 @@
package retry
import (
"context"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/internal/sdk"
)
const (
// DefaultRequestCost is the cost of a single request from the adaptive
// rate limited token bucket.
DefaultRequestCost uint = 1
)
// DefaultThrottles provides the set of errors considered throttle errors that
// are checked by default.
var DefaultThrottles = []IsErrorThrottle{
ThrottleErrorCode{
Codes: DefaultThrottleErrorCodes,
},
}
// AdaptiveModeOptions provides the functional options for configuring the
// adaptive retry mode, and delay behavior.
type AdaptiveModeOptions struct {
// If the adaptive token bucket is empty, when an attempt will be made
// AdaptiveMode will sleep until a token is available. This can occur when
// attempts fail with throttle errors. Use this option to disable the sleep
// until token is available, and return error immediately.
FailOnNoAttemptTokens bool
// The cost of an attempt from the AdaptiveMode's adaptive token bucket.
RequestCost uint
// Set of strategies to determine if the attempt failed due to a throttle
// error.
//
// It is safe to append to this list in NewAdaptiveMode's functional options.
Throttles []IsErrorThrottle
// Set of options for standard retry mode that AdaptiveMode is built on top
// of. AdaptiveMode may apply its own defaults to Standard retry mode that
// are different than the defaults of NewStandard. Use these options to
// override the default options.
StandardOptions []func(*StandardOptions)
}
// AdaptiveMode provides an experimental retry strategy that expands on the
// Standard retry strategy, adding client attempt rate limits. The attempt rate
// limit is initially unrestricted, but becomes restricted when the attempt
// fails with for a throttle error. When restricted AdaptiveMode may need to
// sleep before an attempt is made, if too many throttles have been received.
// AdaptiveMode's sleep can be canceled with context cancel. Set
// AdaptiveModeOptions FailOnNoAttemptTokens to change the behavior from sleep,
// to fail fast.
//
// Eventually unrestricted attempt rate limit will be restored once attempts no
// longer are failing due to throttle errors.
type AdaptiveMode struct {
options AdaptiveModeOptions
throttles IsErrorThrottles
retryer aws.RetryerV2
rateLimit *adaptiveRateLimit
}
// NewAdaptiveMode returns an initialized AdaptiveMode retry strategy.
func NewAdaptiveMode(optFns ...func(*AdaptiveModeOptions)) *AdaptiveMode {
o := AdaptiveModeOptions{
RequestCost: DefaultRequestCost,
Throttles: append([]IsErrorThrottle{}, DefaultThrottles...),
}
for _, fn := range optFns {
fn(&o)
}
return &AdaptiveMode{
options: o,
throttles: IsErrorThrottles(o.Throttles),
retryer: NewStandard(o.StandardOptions...),
rateLimit: newAdaptiveRateLimit(),
}
}
// IsErrorRetryable returns if the failed attempt is retryable. This check
// should determine if the error can be retried, or if the error is
// terminal.
func (a *AdaptiveMode) IsErrorRetryable(err error) bool {
return a.retryer.IsErrorRetryable(err)
}
// MaxAttempts returns the maximum number of attempts that can be made for
// a attempt before failing. A value of 0 implies that the attempt should
// be retried until it succeeds if the errors are retryable.
func (a *AdaptiveMode) MaxAttempts() int {
return a.retryer.MaxAttempts()
}
// RetryDelay returns the delay that should be used before retrying the
// attempt. Will return error if the if the delay could not be determined.
func (a *AdaptiveMode) RetryDelay(attempt int, opErr error) (
time.Duration, error,
) {
return a.retryer.RetryDelay(attempt, opErr)
}
// GetRetryToken attempts to deduct the retry cost from the retry token pool.
// Returning the token release function, or error.
func (a *AdaptiveMode) GetRetryToken(ctx context.Context, opErr error) (
releaseToken func(error) error, err error,
) {
return a.retryer.GetRetryToken(ctx, opErr)
}
// GetInitialToken returns the initial attempt token that can increment the
// retry token pool if the attempt is successful.
//
// Deprecated: This method does not provide a way to block using Context,
// nor can it return an error. Use RetryerV2, and GetAttemptToken instead. Only
// present to implement Retryer interface.
func (a *AdaptiveMode) GetInitialToken() (releaseToken func(error) error) {
return nopRelease
}
// GetAttemptToken returns the attempt token that can be used to rate limit
// attempt calls. Will be used by the SDK's retry package's Attempt
// middleware to get a attempt token prior to calling the temp and releasing
// the attempt token after the attempt has been made.
func (a *AdaptiveMode) GetAttemptToken(ctx context.Context) (func(error) error, error) {
for {
acquiredToken, waitTryAgain := a.rateLimit.AcquireToken(a.options.RequestCost)
if acquiredToken {
break
}
if a.options.FailOnNoAttemptTokens {
return nil, fmt.Errorf(
"unable to get attempt token, and FailOnNoAttemptTokens enables")
}
if err := sdk.SleepWithContext(ctx, waitTryAgain); err != nil {
return nil, fmt.Errorf("failed to wait for token to be available, %w", err)
}
}
return a.handleResponse, nil
}
func (a *AdaptiveMode) handleResponse(opErr error) error {
throttled := a.throttles.IsErrorThrottle(opErr).Bool()
a.rateLimit.Update(throttled)
return nil
}

View File

@@ -0,0 +1,158 @@
package retry
import (
"math"
"sync"
"time"
"github.com/aws/aws-sdk-go-v2/internal/sdk"
)
type adaptiveRateLimit struct {
tokenBucketEnabled bool
smooth float64
beta float64
scaleConstant float64
minFillRate float64
fillRate float64
calculatedRate float64
lastRefilled time.Time
measuredTxRate float64
lastTxRateBucket float64
requestCount int64
lastMaxRate float64
lastThrottleTime time.Time
timeWindow float64
tokenBucket *adaptiveTokenBucket
mu sync.Mutex
}
func newAdaptiveRateLimit() *adaptiveRateLimit {
now := sdk.NowTime()
return &adaptiveRateLimit{
smooth: 0.8,
beta: 0.7,
scaleConstant: 0.4,
minFillRate: 0.5,
lastTxRateBucket: math.Floor(timeFloat64Seconds(now)),
lastThrottleTime: now,
tokenBucket: newAdaptiveTokenBucket(0),
}
}
func (a *adaptiveRateLimit) Enable(v bool) {
a.mu.Lock()
defer a.mu.Unlock()
a.tokenBucketEnabled = v
}
func (a *adaptiveRateLimit) AcquireToken(amount uint) (
tokenAcquired bool, waitTryAgain time.Duration,
) {
a.mu.Lock()
defer a.mu.Unlock()
if !a.tokenBucketEnabled {
return true, 0
}
a.tokenBucketRefill()
available, ok := a.tokenBucket.Retrieve(float64(amount))
if !ok {
waitDur := float64Seconds((float64(amount) - available) / a.fillRate)
return false, waitDur
}
return true, 0
}
func (a *adaptiveRateLimit) Update(throttled bool) {
a.mu.Lock()
defer a.mu.Unlock()
a.updateMeasuredRate()
if throttled {
rateToUse := a.measuredTxRate
if a.tokenBucketEnabled {
rateToUse = math.Min(a.measuredTxRate, a.fillRate)
}
a.lastMaxRate = rateToUse
a.calculateTimeWindow()
a.lastThrottleTime = sdk.NowTime()
a.calculatedRate = a.cubicThrottle(rateToUse)
a.tokenBucketEnabled = true
} else {
a.calculateTimeWindow()
a.calculatedRate = a.cubicSuccess(sdk.NowTime())
}
newRate := math.Min(a.calculatedRate, 2*a.measuredTxRate)
a.tokenBucketUpdateRate(newRate)
}
func (a *adaptiveRateLimit) cubicSuccess(t time.Time) float64 {
dt := secondsFloat64(t.Sub(a.lastThrottleTime))
return (a.scaleConstant * math.Pow(dt-a.timeWindow, 3)) + a.lastMaxRate
}
func (a *adaptiveRateLimit) cubicThrottle(rateToUse float64) float64 {
return rateToUse * a.beta
}
func (a *adaptiveRateLimit) calculateTimeWindow() {
a.timeWindow = math.Pow((a.lastMaxRate*(1.-a.beta))/a.scaleConstant, 1./3.)
}
func (a *adaptiveRateLimit) tokenBucketUpdateRate(newRPS float64) {
a.tokenBucketRefill()
a.fillRate = math.Max(newRPS, a.minFillRate)
a.tokenBucket.Resize(newRPS)
}
func (a *adaptiveRateLimit) updateMeasuredRate() {
now := sdk.NowTime()
timeBucket := math.Floor(timeFloat64Seconds(now)*2.) / 2.
a.requestCount++
if timeBucket > a.lastTxRateBucket {
currentRate := float64(a.requestCount) / (timeBucket - a.lastTxRateBucket)
a.measuredTxRate = (currentRate * a.smooth) + (a.measuredTxRate * (1. - a.smooth))
a.requestCount = 0
a.lastTxRateBucket = timeBucket
}
}
func (a *adaptiveRateLimit) tokenBucketRefill() {
now := sdk.NowTime()
if a.lastRefilled.IsZero() {
a.lastRefilled = now
return
}
fillAmount := secondsFloat64(now.Sub(a.lastRefilled)) * a.fillRate
a.tokenBucket.Refund(fillAmount)
a.lastRefilled = now
}
func float64Seconds(v float64) time.Duration {
return time.Duration(v * float64(time.Second))
}
func secondsFloat64(v time.Duration) float64 {
return float64(v) / float64(time.Second)
}
func timeFloat64Seconds(v time.Time) float64 {
return float64(v.UnixNano()) / float64(time.Second)
}

View File

@@ -0,0 +1,83 @@
package retry
import (
"math"
"sync"
)
// adaptiveTokenBucket provides a concurrency safe utility for adding and
// removing tokens from the available token bucket.
type adaptiveTokenBucket struct {
remainingTokens float64
maxCapacity float64
minCapacity float64
mu sync.Mutex
}
// newAdaptiveTokenBucket returns an initialized adaptiveTokenBucket with the
// capacity specified.
func newAdaptiveTokenBucket(i float64) *adaptiveTokenBucket {
return &adaptiveTokenBucket{
remainingTokens: i,
maxCapacity: i,
minCapacity: 1,
}
}
// Retrieve attempts to reduce the available tokens by the amount requested. If
// there are tokens available true will be returned along with the number of
// available tokens remaining. If amount requested is larger than the available
// capacity, false will be returned along with the available capacity. If the
// amount is less than the available capacity, the capacity will be reduced by
// that amount, and the remaining capacity and true will be returned.
func (t *adaptiveTokenBucket) Retrieve(amount float64) (available float64, retrieved bool) {
t.mu.Lock()
defer t.mu.Unlock()
if amount > t.remainingTokens {
return t.remainingTokens, false
}
t.remainingTokens -= amount
return t.remainingTokens, true
}
// Refund returns the amount of tokens back to the available token bucket, up
// to the initial capacity.
func (t *adaptiveTokenBucket) Refund(amount float64) {
t.mu.Lock()
defer t.mu.Unlock()
// Capacity cannot exceed max capacity.
t.remainingTokens = math.Min(t.remainingTokens+amount, t.maxCapacity)
}
// Capacity returns the maximum capacity of tokens that the bucket could
// contain.
func (t *adaptiveTokenBucket) Capacity() float64 {
t.mu.Lock()
defer t.mu.Unlock()
return t.maxCapacity
}
// Remaining returns the number of tokens that remaining in the bucket.
func (t *adaptiveTokenBucket) Remaining() float64 {
t.mu.Lock()
defer t.mu.Unlock()
return t.remainingTokens
}
// Resize adjusts the size of the token bucket. Returns the capacity remaining.
func (t *adaptiveTokenBucket) Resize(size float64) float64 {
t.mu.Lock()
defer t.mu.Unlock()
t.maxCapacity = math.Max(size, t.minCapacity)
// Capacity needs to be capped at max capacity, if max size reduced.
t.remainingTokens = math.Min(t.remainingTokens, t.maxCapacity)
return t.remainingTokens
}

80
vendor/github.com/aws/aws-sdk-go-v2/aws/retry/doc.go generated vendored Normal file
View File

@@ -0,0 +1,80 @@
// Package retry provides interfaces and implementations for SDK request retry behavior.
//
// Retryer Interface and Implementations
//
// This packages defines Retryer interface that is used to either implement custom retry behavior
// or to extend the existing retry implementations provided by the SDK. This packages provides a single
// retry implementations: Standard.
//
// Standard
//
// Standard is the default retryer implementation used by service clients. The standard retryer is a rate limited
// retryer that has a configurable max attempts to limit the number of retry attempts when a retryable error occurs.
// In addition, the retryer uses a configurable token bucket to rate limit the retry attempts across the client,
// and uses an additional delay policy to limit the time between a requests subsequent attempts.
//
// By default the standard retryer uses the DefaultRetryables slice of IsErrorRetryable types to determine whether
// a given error is retryable. By default this list of retryables includes the following:
// - Retrying errors that implement the RetryableError method, and return true.
// - Connection Errors
// - Errors that implement a ConnectionError, Temporary, or Timeout method that return true.
// - Connection Reset Errors.
// - net.OpErr types that are dialing errors or are temporary.
// - HTTP Status Codes: 500, 502, 503, and 504.
// - API Error Codes
// - RequestTimeout, RequestTimeoutException
// - Throttling, ThrottlingException, ThrottledException, RequestThrottledException, TooManyRequestsException,
// RequestThrottled, SlowDown, EC2ThrottledException
// - ProvisionedThroughputExceededException, RequestLimitExceeded, BandwidthLimitExceeded, LimitExceededException
// - TransactionInProgressException, PriorRequestNotComplete
//
// The standard retryer will not retry a request in the event if the context associated with the request
// has been cancelled. Applications must handle this case explicitly if they wish to retry with a different context
// value.
//
// You can configure the standard retryer implementation to fit your applications by constructing a standard retryer
// using the NewStandard function, and providing one more functional arguments that mutate the StandardOptions
// structure. StandardOptions provides the ability to modify the token bucket rate limiter, retryable error conditions,
// and the retry delay policy.
//
// For example to modify the default retry attempts for the standard retryer:
//
// // configure the custom retryer
// customRetry := retry.NewStandard(func(o *retry.StandardOptions) {
// o.MaxAttempts = 5
// })
//
// // create a service client with the retryer
// s3.NewFromConfig(cfg, func(o *s3.Options) {
// o.Retryer = customRetry
// })
//
// Utilities
//
// A number of package functions have been provided to easily wrap retryer implementations in an implementation agnostic
// way. These are:
//
// AddWithErrorCodes - Provides the ability to add additional API error codes that should be considered retryable
// in addition to those considered retryable by the provided retryer.
//
// AddWithMaxAttempts - Provides the ability to set the max number of attempts for retrying a request by wrapping
// a retryer implementation.
//
// AddWithMaxBackoffDelay - Provides the ability to set the max back off delay that can occur before retrying a
// request by wrapping a retryer implementation.
//
// The following package functions have been provided to easily satisfy different retry interfaces to further customize
// a given retryer's behavior:
//
// BackoffDelayerFunc - Can be used to wrap a function to satisfy the BackoffDelayer interface. For example,
// you can use this method to easily create custom back off policies to be used with the
// standard retryer.
//
// IsErrorRetryableFunc - Can be used to wrap a function to satisfy the IsErrorRetryable interface. For example,
// this can be used to extend the standard retryer to add additional logic ot determine if a
// error should be retried.
//
// IsErrorTimeoutFunc - Can be used to wrap a function to satisfy IsErrorTimeout interface. For example,
// this can be used to extend the standard retryer to add additional logic to determine if an
// error should be considered a timeout.
package retry

View File

@@ -0,0 +1,20 @@
package retry
import "fmt"
// MaxAttemptsError provides the error when the maximum number of attempts have
// been exceeded.
type MaxAttemptsError struct {
Attempt int
Err error
}
func (e *MaxAttemptsError) Error() string {
return fmt.Sprintf("exceeded maximum number of attempts, %d, %v", e.Attempt, e.Err)
}
// Unwrap returns the nested error causing the max attempts error. Provides the
// implementation for errors.Is and errors.As to unwrap nested errors.
func (e *MaxAttemptsError) Unwrap() error {
return e.Err
}

View File

@@ -0,0 +1,49 @@
package retry
import (
"math"
"time"
"github.com/aws/aws-sdk-go-v2/internal/rand"
"github.com/aws/aws-sdk-go-v2/internal/timeconv"
)
// ExponentialJitterBackoff provides backoff delays with jitter based on the
// number of attempts.
type ExponentialJitterBackoff struct {
maxBackoff time.Duration
// precomputed number of attempts needed to reach max backoff.
maxBackoffAttempts float64
randFloat64 func() (float64, error)
}
// NewExponentialJitterBackoff returns an ExponentialJitterBackoff configured
// for the max backoff.
func NewExponentialJitterBackoff(maxBackoff time.Duration) *ExponentialJitterBackoff {
return &ExponentialJitterBackoff{
maxBackoff: maxBackoff,
maxBackoffAttempts: math.Log2(
float64(maxBackoff) / float64(time.Second)),
randFloat64: rand.CryptoRandFloat64,
}
}
// BackoffDelay returns the duration to wait before the next attempt should be
// made. Returns an error if unable get a duration.
func (j *ExponentialJitterBackoff) BackoffDelay(attempt int, err error) (time.Duration, error) {
if attempt > int(j.maxBackoffAttempts) {
return j.maxBackoff, nil
}
b, err := j.randFloat64()
if err != nil {
return 0, err
}
// [0.0, 1.0) * 2 ^ attempts
ri := int64(1 << uint64(attempt))
delaySeconds := b * float64(ri)
return timeconv.FloatSecondsDur(delaySeconds), nil
}

View File

@@ -0,0 +1,52 @@
package retry
import (
awsmiddle "github.com/aws/aws-sdk-go-v2/aws/middleware"
"github.com/aws/smithy-go/middleware"
)
// attemptResultsKey is a metadata accessor key to retrieve metadata
// for all request attempts.
type attemptResultsKey struct {
}
// GetAttemptResults retrieves attempts results from middleware metadata.
func GetAttemptResults(metadata middleware.Metadata) (AttemptResults, bool) {
m, ok := metadata.Get(attemptResultsKey{}).(AttemptResults)
return m, ok
}
// AttemptResults represents struct containing metadata returned by all request attempts.
type AttemptResults struct {
// Results is a slice consisting attempt result from all request attempts.
// Results are stored in order request attempt is made.
Results []AttemptResult
}
// AttemptResult represents attempt result returned by a single request attempt.
type AttemptResult struct {
// Err is the error if received for the request attempt.
Err error
// Retryable denotes if request may be retried. This states if an
// error is considered retryable.
Retryable bool
// Retried indicates if this request was retried.
Retried bool
// ResponseMetadata is any existing metadata passed via the response middlewares.
ResponseMetadata middleware.Metadata
}
// addAttemptResults adds attempt results to middleware metadata
func addAttemptResults(metadata *middleware.Metadata, v AttemptResults) {
metadata.Set(attemptResultsKey{}, v)
}
// GetRawResponse returns raw response recorded for the attempt result
func (a AttemptResult) GetRawResponse() interface{} {
return awsmiddle.GetRawResponse(a.ResponseMetadata)
}

View File

@@ -0,0 +1,331 @@
package retry
import (
"context"
"fmt"
"strconv"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsmiddle "github.com/aws/aws-sdk-go-v2/aws/middleware"
"github.com/aws/aws-sdk-go-v2/internal/sdk"
"github.com/aws/smithy-go/logging"
"github.com/aws/smithy-go/middleware"
smithymiddle "github.com/aws/smithy-go/middleware"
"github.com/aws/smithy-go/transport/http"
)
// RequestCloner is a function that can take an input request type and clone
// the request for use in a subsequent retry attempt.
type RequestCloner func(interface{}) interface{}
type retryMetadata struct {
AttemptNum int
AttemptTime time.Time
MaxAttempts int
AttemptClockSkew time.Duration
}
// Attempt is a Smithy Finalize middleware that handles retry attempts using
// the provided Retryer implementation.
type Attempt struct {
// Enable the logging of retry attempts performed by the SDK. This will
// include logging retry attempts, unretryable errors, and when max
// attempts are reached.
LogAttempts bool
retryer aws.RetryerV2
requestCloner RequestCloner
}
// NewAttemptMiddleware returns a new Attempt retry middleware.
func NewAttemptMiddleware(retryer aws.Retryer, requestCloner RequestCloner, optFns ...func(*Attempt)) *Attempt {
m := &Attempt{
retryer: wrapAsRetryerV2(retryer),
requestCloner: requestCloner,
}
for _, fn := range optFns {
fn(m)
}
return m
}
// ID returns the middleware identifier
func (r *Attempt) ID() string { return "Retry" }
func (r Attempt) logf(logger logging.Logger, classification logging.Classification, format string, v ...interface{}) {
if !r.LogAttempts {
return
}
logger.Logf(classification, format, v...)
}
// HandleFinalize utilizes the provider Retryer implementation to attempt
// retries over the next handler
func (r *Attempt) HandleFinalize(ctx context.Context, in smithymiddle.FinalizeInput, next smithymiddle.FinalizeHandler) (
out smithymiddle.FinalizeOutput, metadata smithymiddle.Metadata, err error,
) {
var attemptNum int
var attemptClockSkew time.Duration
var attemptResults AttemptResults
maxAttempts := r.retryer.MaxAttempts()
releaseRetryToken := nopRelease
for {
attemptNum++
attemptInput := in
attemptInput.Request = r.requestCloner(attemptInput.Request)
// Record the metadata for the for attempt being started.
attemptCtx := setRetryMetadata(ctx, retryMetadata{
AttemptNum: attemptNum,
AttemptTime: sdk.NowTime().UTC(),
MaxAttempts: maxAttempts,
AttemptClockSkew: attemptClockSkew,
})
var attemptResult AttemptResult
out, attemptResult, releaseRetryToken, err = r.handleAttempt(attemptCtx, attemptInput, releaseRetryToken, next)
attemptClockSkew, _ = awsmiddle.GetAttemptSkew(attemptResult.ResponseMetadata)
// AttempResult Retried states that the attempt was not successful, and
// should be retried.
shouldRetry := attemptResult.Retried
// Add attempt metadata to list of all attempt metadata
attemptResults.Results = append(attemptResults.Results, attemptResult)
if !shouldRetry {
// Ensure the last response's metadata is used as the bases for result
// metadata returned by the stack. The Slice of attempt results
// will be added to this cloned metadata.
metadata = attemptResult.ResponseMetadata.Clone()
break
}
}
addAttemptResults(&metadata, attemptResults)
return out, metadata, err
}
// handleAttempt handles an individual request attempt.
func (r *Attempt) handleAttempt(
ctx context.Context, in smithymiddle.FinalizeInput, releaseRetryToken func(error) error, next smithymiddle.FinalizeHandler,
) (
out smithymiddle.FinalizeOutput, attemptResult AttemptResult, _ func(error) error, err error,
) {
defer func() {
attemptResult.Err = err
}()
// Short circuit if this attempt never can succeed because the context is
// canceled. This reduces the chance of token pools being modified for
// attempts that will not be made
select {
case <-ctx.Done():
return out, attemptResult, nopRelease, ctx.Err()
default:
}
//------------------------------
// Get Attempt Token
//------------------------------
releaseAttemptToken, err := r.retryer.GetAttemptToken(ctx)
if err != nil {
return out, attemptResult, nopRelease, fmt.Errorf(
"failed to get retry Send token, %w", err)
}
//------------------------------
// Send Attempt
//------------------------------
logger := smithymiddle.GetLogger(ctx)
service, operation := awsmiddle.GetServiceID(ctx), awsmiddle.GetOperationName(ctx)
retryMetadata, _ := getRetryMetadata(ctx)
attemptNum := retryMetadata.AttemptNum
maxAttempts := retryMetadata.MaxAttempts
// Following attempts must ensure the request payload stream starts in a
// rewound state.
if attemptNum > 1 {
if rewindable, ok := in.Request.(interface{ RewindStream() error }); ok {
if rewindErr := rewindable.RewindStream(); rewindErr != nil {
return out, attemptResult, nopRelease, fmt.Errorf(
"failed to rewind transport stream for retry, %w", rewindErr)
}
}
r.logf(logger, logging.Debug, "retrying request %s/%s, attempt %d",
service, operation, attemptNum)
}
var metadata smithymiddle.Metadata
out, metadata, err = next.HandleFinalize(ctx, in)
attemptResult.ResponseMetadata = metadata
//------------------------------
// Bookkeeping
//------------------------------
// Release the retry token based on the state of the attempt's error (if any).
if releaseError := releaseRetryToken(err); releaseError != nil && err != nil {
return out, attemptResult, nopRelease, fmt.Errorf(
"failed to release retry token after request error, %w", err)
}
// Release the attempt token based on the state of the attempt's error (if any).
if releaseError := releaseAttemptToken(err); releaseError != nil && err != nil {
return out, attemptResult, nopRelease, fmt.Errorf(
"failed to release initial token after request error, %w", err)
}
// If there was no error making the attempt, nothing further to do. There
// will be nothing to retry.
if err == nil {
return out, attemptResult, nopRelease, err
}
//------------------------------
// Is Retryable and Should Retry
//------------------------------
// If the attempt failed with an unretryable error, nothing further to do
// but return, and inform the caller about the terminal failure.
retryable := r.retryer.IsErrorRetryable(err)
if !retryable {
r.logf(logger, logging.Debug, "request failed with unretryable error %v", err)
return out, attemptResult, nopRelease, err
}
// set retryable to true
attemptResult.Retryable = true
// Once the maximum number of attempts have been exhausted there is nothing
// further to do other than inform the caller about the terminal failure.
if maxAttempts > 0 && attemptNum >= maxAttempts {
r.logf(logger, logging.Debug, "max retry attempts exhausted, max %d", maxAttempts)
err = &MaxAttemptsError{
Attempt: attemptNum,
Err: err,
}
return out, attemptResult, nopRelease, err
}
//------------------------------
// Get Retry (aka Retry Quota) Token
//------------------------------
// Get a retry token that will be released after the
releaseRetryToken, retryTokenErr := r.retryer.GetRetryToken(ctx, err)
if retryTokenErr != nil {
return out, attemptResult, nopRelease, retryTokenErr
}
//------------------------------
// Retry Delay and Sleep
//------------------------------
// Get the retry delay before another attempt can be made, and sleep for
// that time. Potentially early exist if the sleep is canceled via the
// context.
retryDelay, reqErr := r.retryer.RetryDelay(attemptNum, err)
if reqErr != nil {
return out, attemptResult, releaseRetryToken, reqErr
}
if reqErr = sdk.SleepWithContext(ctx, retryDelay); reqErr != nil {
err = &aws.RequestCanceledError{Err: reqErr}
return out, attemptResult, releaseRetryToken, err
}
// The request should be re-attempted.
attemptResult.Retried = true
return out, attemptResult, releaseRetryToken, err
}
// MetricsHeader attaches SDK request metric header for retries to the transport
type MetricsHeader struct{}
// ID returns the middleware identifier
func (r *MetricsHeader) ID() string {
return "RetryMetricsHeader"
}
// HandleFinalize attaches the SDK request metric header to the transport layer
func (r MetricsHeader) HandleFinalize(ctx context.Context, in smithymiddle.FinalizeInput, next smithymiddle.FinalizeHandler) (
out smithymiddle.FinalizeOutput, metadata smithymiddle.Metadata, err error,
) {
retryMetadata, _ := getRetryMetadata(ctx)
const retryMetricHeader = "Amz-Sdk-Request"
var parts []string
parts = append(parts, "attempt="+strconv.Itoa(retryMetadata.AttemptNum))
if retryMetadata.MaxAttempts != 0 {
parts = append(parts, "max="+strconv.Itoa(retryMetadata.MaxAttempts))
}
var ttl time.Time
if deadline, ok := ctx.Deadline(); ok {
ttl = deadline
}
// Only append the TTL if it can be determined.
if !ttl.IsZero() && retryMetadata.AttemptClockSkew > 0 {
const unixTimeFormat = "20060102T150405Z"
ttl = ttl.Add(retryMetadata.AttemptClockSkew)
parts = append(parts, "ttl="+ttl.Format(unixTimeFormat))
}
switch req := in.Request.(type) {
case *http.Request:
req.Header[retryMetricHeader] = append(req.Header[retryMetricHeader][:0], strings.Join(parts, "; "))
default:
return out, metadata, fmt.Errorf("unknown transport type %T", req)
}
return next.HandleFinalize(ctx, in)
}
type retryMetadataKey struct{}
// getRetryMetadata retrieves retryMetadata from the context and a bool
// indicating if it was set.
//
// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
// to clear all stack values.
func getRetryMetadata(ctx context.Context) (metadata retryMetadata, ok bool) {
metadata, ok = middleware.GetStackValue(ctx, retryMetadataKey{}).(retryMetadata)
return metadata, ok
}
// setRetryMetadata sets the retryMetadata on the context.
//
// Scoped to stack values. Use github.com/aws/smithy-go/middleware#ClearStackValues
// to clear all stack values.
func setRetryMetadata(ctx context.Context, metadata retryMetadata) context.Context {
return middleware.WithStackValue(ctx, retryMetadataKey{}, metadata)
}
// AddRetryMiddlewaresOptions is the set of options that can be passed to
// AddRetryMiddlewares for configuring retry associated middleware.
type AddRetryMiddlewaresOptions struct {
Retryer aws.Retryer
// Enable the logging of retry attempts performed by the SDK. This will
// include logging retry attempts, unretryable errors, and when max
// attempts are reached.
LogRetryAttempts bool
}
// AddRetryMiddlewares adds retry middleware to operation middleware stack
func AddRetryMiddlewares(stack *smithymiddle.Stack, options AddRetryMiddlewaresOptions) error {
attempt := NewAttemptMiddleware(options.Retryer, http.RequestCloner, func(middleware *Attempt) {
middleware.LogAttempts = options.LogRetryAttempts
})
if err := stack.Finalize.Add(attempt, smithymiddle.After); err != nil {
return err
}
if err := stack.Finalize.Add(&MetricsHeader{}, smithymiddle.After); err != nil {
return err
}
return nil
}

90
vendor/github.com/aws/aws-sdk-go-v2/aws/retry/retry.go generated vendored Normal file
View File

@@ -0,0 +1,90 @@
package retry
import (
"context"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
)
// AddWithErrorCodes returns a Retryer with additional error codes considered
// for determining if the error should be retried.
func AddWithErrorCodes(r aws.Retryer, codes ...string) aws.Retryer {
retryable := &RetryableErrorCode{
Codes: map[string]struct{}{},
}
for _, c := range codes {
retryable.Codes[c] = struct{}{}
}
return &withIsErrorRetryable{
RetryerV2: wrapAsRetryerV2(r),
Retryable: retryable,
}
}
type withIsErrorRetryable struct {
aws.RetryerV2
Retryable IsErrorRetryable
}
func (r *withIsErrorRetryable) IsErrorRetryable(err error) bool {
if v := r.Retryable.IsErrorRetryable(err); v != aws.UnknownTernary {
return v.Bool()
}
return r.RetryerV2.IsErrorRetryable(err)
}
// AddWithMaxAttempts returns a Retryer with MaxAttempts set to the value
// specified.
func AddWithMaxAttempts(r aws.Retryer, max int) aws.Retryer {
return &withMaxAttempts{
RetryerV2: wrapAsRetryerV2(r),
Max: max,
}
}
type withMaxAttempts struct {
aws.RetryerV2
Max int
}
func (w *withMaxAttempts) MaxAttempts() int {
return w.Max
}
// AddWithMaxBackoffDelay returns a retryer wrapping the passed in retryer
// overriding the RetryDelay behavior for a alternate minimum initial backoff
// delay.
func AddWithMaxBackoffDelay(r aws.Retryer, delay time.Duration) aws.Retryer {
return &withMaxBackoffDelay{
RetryerV2: wrapAsRetryerV2(r),
backoff: NewExponentialJitterBackoff(delay),
}
}
type withMaxBackoffDelay struct {
aws.RetryerV2
backoff *ExponentialJitterBackoff
}
func (r *withMaxBackoffDelay) RetryDelay(attempt int, err error) (time.Duration, error) {
return r.backoff.BackoffDelay(attempt, err)
}
type wrappedAsRetryerV2 struct {
aws.Retryer
}
func wrapAsRetryerV2(r aws.Retryer) aws.RetryerV2 {
v, ok := r.(aws.RetryerV2)
if !ok {
v = wrappedAsRetryerV2{Retryer: r}
}
return v
}
func (w wrappedAsRetryerV2) GetAttemptToken(context.Context) (func(error) error, error) {
return w.Retryer.GetInitialToken(), nil
}

View File

@@ -0,0 +1,186 @@
package retry
import (
"errors"
"net"
"net/url"
"strings"
"github.com/aws/aws-sdk-go-v2/aws"
)
// IsErrorRetryable provides the interface of an implementation to determine if
// a error as the result of an operation is retryable.
type IsErrorRetryable interface {
IsErrorRetryable(error) aws.Ternary
}
// IsErrorRetryables is a collection of checks to determine of the error is
// retryable. Iterates through the checks and returns the state of retryable
// if any check returns something other than unknown.
type IsErrorRetryables []IsErrorRetryable
// IsErrorRetryable returns if the error is retryable if any of the checks in
// the list return a value other than unknown.
func (r IsErrorRetryables) IsErrorRetryable(err error) aws.Ternary {
for _, re := range r {
if v := re.IsErrorRetryable(err); v != aws.UnknownTernary {
return v
}
}
return aws.UnknownTernary
}
// IsErrorRetryableFunc wraps a function with the IsErrorRetryable interface.
type IsErrorRetryableFunc func(error) aws.Ternary
// IsErrorRetryable returns if the error is retryable.
func (fn IsErrorRetryableFunc) IsErrorRetryable(err error) aws.Ternary {
return fn(err)
}
// RetryableError is an IsErrorRetryable implementation which uses the
// optional interface Retryable on the error value to determine if the error is
// retryable.
type RetryableError struct{}
// IsErrorRetryable returns if the error is retryable if it satisfies the
// Retryable interface, and returns if the attempt should be retried.
func (RetryableError) IsErrorRetryable(err error) aws.Ternary {
var v interface{ RetryableError() bool }
if !errors.As(err, &v) {
return aws.UnknownTernary
}
return aws.BoolTernary(v.RetryableError())
}
// NoRetryCanceledError detects if the error was an request canceled error and
// returns if so.
type NoRetryCanceledError struct{}
// IsErrorRetryable returns the error is not retryable if the request was
// canceled.
func (NoRetryCanceledError) IsErrorRetryable(err error) aws.Ternary {
var v interface{ CanceledError() bool }
if !errors.As(err, &v) {
return aws.UnknownTernary
}
if v.CanceledError() {
return aws.FalseTernary
}
return aws.UnknownTernary
}
// RetryableConnectionError determines if the underlying error is an HTTP
// connection and returns if it should be retried.
//
// Includes errors such as connection reset, connection refused, net dial,
// temporary, and timeout errors.
type RetryableConnectionError struct{}
// IsErrorRetryable returns if the error is caused by and HTTP connection
// error, and should be retried.
func (r RetryableConnectionError) IsErrorRetryable(err error) aws.Ternary {
if err == nil {
return aws.UnknownTernary
}
var retryable bool
var conErr interface{ ConnectionError() bool }
var tempErr interface{ Temporary() bool }
var timeoutErr interface{ Timeout() bool }
var urlErr *url.Error
var netOpErr *net.OpError
switch {
case errors.As(err, &conErr) && conErr.ConnectionError():
retryable = true
case strings.Contains(err.Error(), "connection reset"):
retryable = true
case errors.As(err, &urlErr):
// Refused connections should be retried as the service may not yet be
// running on the port. Go TCP dial considers refused connections as
// not temporary.
if strings.Contains(urlErr.Error(), "connection refused") {
retryable = true
} else {
return r.IsErrorRetryable(errors.Unwrap(urlErr))
}
case errors.As(err, &netOpErr):
// Network dial, or temporary network errors are always retryable.
if strings.EqualFold(netOpErr.Op, "dial") || netOpErr.Temporary() {
retryable = true
} else {
return r.IsErrorRetryable(errors.Unwrap(netOpErr))
}
case errors.As(err, &tempErr) && tempErr.Temporary():
// Fallback to the generic temporary check, with temporary errors
// retryable.
retryable = true
case errors.As(err, &timeoutErr) && timeoutErr.Timeout():
// Fallback to the generic timeout check, with timeout errors
// retryable.
retryable = true
default:
return aws.UnknownTernary
}
return aws.BoolTernary(retryable)
}
// RetryableHTTPStatusCode provides a IsErrorRetryable based on HTTP status
// codes.
type RetryableHTTPStatusCode struct {
Codes map[int]struct{}
}
// IsErrorRetryable return if the passed in error is retryable based on the
// HTTP status code.
func (r RetryableHTTPStatusCode) IsErrorRetryable(err error) aws.Ternary {
var v interface{ HTTPStatusCode() int }
if !errors.As(err, &v) {
return aws.UnknownTernary
}
_, ok := r.Codes[v.HTTPStatusCode()]
if !ok {
return aws.UnknownTernary
}
return aws.TrueTernary
}
// RetryableErrorCode determines if an attempt should be retried based on the
// API error code.
type RetryableErrorCode struct {
Codes map[string]struct{}
}
// IsErrorRetryable return if the error is retryable based on the error codes.
// Returns unknown if the error doesn't have a code or it is unknown.
func (r RetryableErrorCode) IsErrorRetryable(err error) aws.Ternary {
var v interface{ ErrorCode() string }
if !errors.As(err, &v) {
return aws.UnknownTernary
}
_, ok := r.Codes[v.ErrorCode()]
if !ok {
return aws.UnknownTernary
}
return aws.TrueTernary
}

View File

@@ -0,0 +1,258 @@
package retry
import (
"context"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws/ratelimit"
)
// BackoffDelayer provides the interface for determining the delay to before
// another request attempt, that previously failed.
type BackoffDelayer interface {
BackoffDelay(attempt int, err error) (time.Duration, error)
}
// BackoffDelayerFunc provides a wrapper around a function to determine the
// backoff delay of an attempt retry.
type BackoffDelayerFunc func(int, error) (time.Duration, error)
// BackoffDelay returns the delay before attempt to retry a request.
func (fn BackoffDelayerFunc) BackoffDelay(attempt int, err error) (time.Duration, error) {
return fn(attempt, err)
}
const (
// DefaultMaxAttempts is the maximum of attempts for an API request
DefaultMaxAttempts int = 3
// DefaultMaxBackoff is the maximum back off delay between attempts
DefaultMaxBackoff time.Duration = 20 * time.Second
)
// Default retry token quota values.
const (
DefaultRetryRateTokens uint = 500
DefaultRetryCost uint = 5
DefaultRetryTimeoutCost uint = 10
DefaultNoRetryIncrement uint = 1
)
// DefaultRetryableHTTPStatusCodes is the default set of HTTP status codes the SDK
// should consider as retryable errors.
var DefaultRetryableHTTPStatusCodes = map[int]struct{}{
500: {},
502: {},
503: {},
504: {},
}
// DefaultRetryableErrorCodes provides the set of API error codes that should
// be retried.
var DefaultRetryableErrorCodes = map[string]struct{}{
"RequestTimeout": {},
"RequestTimeoutException": {},
}
// DefaultThrottleErrorCodes provides the set of API error codes that are
// considered throttle errors.
var DefaultThrottleErrorCodes = map[string]struct{}{
"Throttling": {},
"ThrottlingException": {},
"ThrottledException": {},
"RequestThrottledException": {},
"TooManyRequestsException": {},
"ProvisionedThroughputExceededException": {},
"TransactionInProgressException": {},
"RequestLimitExceeded": {},
"BandwidthLimitExceeded": {},
"LimitExceededException": {},
"RequestThrottled": {},
"SlowDown": {},
"PriorRequestNotComplete": {},
"EC2ThrottledException": {},
}
// DefaultRetryables provides the set of retryable checks that are used by
// default.
var DefaultRetryables = []IsErrorRetryable{
NoRetryCanceledError{},
RetryableError{},
RetryableConnectionError{},
RetryableHTTPStatusCode{
Codes: DefaultRetryableHTTPStatusCodes,
},
RetryableErrorCode{
Codes: DefaultRetryableErrorCodes,
},
RetryableErrorCode{
Codes: DefaultThrottleErrorCodes,
},
}
// DefaultTimeouts provides the set of timeout checks that are used by default.
var DefaultTimeouts = []IsErrorTimeout{
TimeouterError{},
}
// StandardOptions provides the functional options for configuring the standard
// retryable, and delay behavior.
type StandardOptions struct {
// Maximum number of attempts that should be made.
MaxAttempts int
// MaxBackoff duration between retried attempts.
MaxBackoff time.Duration
// Provides the backoff strategy the retryer will use to determine the
// delay between retry attempts.
Backoff BackoffDelayer
// Set of strategies to determine if the attempt should be retried based on
// the error response received.
//
// It is safe to append to this list in NewStandard's functional options.
Retryables []IsErrorRetryable
// Set of strategies to determine if the attempt failed due to a timeout
// error.
//
// It is safe to append to this list in NewStandard's functional options.
Timeouts []IsErrorTimeout
// Provides the rate limiting strategy for rate limiting attempt retries
// across all attempts the retryer is being used with.
RateLimiter RateLimiter
// The cost to deduct from the RateLimiter's token bucket per retry.
RetryCost uint
// The cost to deduct from the RateLimiter's token bucket per retry caused
// by timeout error.
RetryTimeoutCost uint
// The cost to payback to the RateLimiter's token bucket for successful
// attempts.
NoRetryIncrement uint
}
// RateLimiter provides the interface for limiting the rate of attempt retries
// allowed by the retryer.
type RateLimiter interface {
GetToken(ctx context.Context, cost uint) (releaseToken func() error, err error)
AddTokens(uint) error
}
// Standard is the standard retry pattern for the SDK. It uses a set of
// retryable checks to determine of the failed attempt should be retried, and
// what retry delay should be used.
type Standard struct {
options StandardOptions
timeout IsErrorTimeout
retryable IsErrorRetryable
backoff BackoffDelayer
}
// NewStandard initializes a standard retry behavior with defaults that can be
// overridden via functional options.
func NewStandard(fnOpts ...func(*StandardOptions)) *Standard {
o := StandardOptions{
MaxAttempts: DefaultMaxAttempts,
MaxBackoff: DefaultMaxBackoff,
Retryables: append([]IsErrorRetryable{}, DefaultRetryables...),
Timeouts: append([]IsErrorTimeout{}, DefaultTimeouts...),
RateLimiter: ratelimit.NewTokenRateLimit(DefaultRetryRateTokens),
RetryCost: DefaultRetryCost,
RetryTimeoutCost: DefaultRetryTimeoutCost,
NoRetryIncrement: DefaultNoRetryIncrement,
}
for _, fn := range fnOpts {
fn(&o)
}
if o.MaxAttempts <= 0 {
o.MaxAttempts = DefaultMaxAttempts
}
backoff := o.Backoff
if backoff == nil {
backoff = NewExponentialJitterBackoff(o.MaxBackoff)
}
return &Standard{
options: o,
backoff: backoff,
retryable: IsErrorRetryables(o.Retryables),
timeout: IsErrorTimeouts(o.Timeouts),
}
}
// MaxAttempts returns the maximum number of attempts that can be made for a
// request before failing.
func (s *Standard) MaxAttempts() int {
return s.options.MaxAttempts
}
// IsErrorRetryable returns if the error is can be retried or not. Should not
// consider the number of attempts made.
func (s *Standard) IsErrorRetryable(err error) bool {
return s.retryable.IsErrorRetryable(err).Bool()
}
// RetryDelay returns the delay to use before another request attempt is made.
func (s *Standard) RetryDelay(attempt int, err error) (time.Duration, error) {
return s.backoff.BackoffDelay(attempt, err)
}
// GetAttemptToken returns the token to be released after then attempt completes.
// The release token will add NoRetryIncrement to the RateLimiter token pool if
// the attempt was successful. If the attempt failed, nothing will be done.
func (s *Standard) GetAttemptToken(context.Context) (func(error) error, error) {
return s.GetInitialToken(), nil
}
// GetInitialToken returns a token for adding the NoRetryIncrement to the
// RateLimiter token if the attempt completed successfully without error.
//
// InitialToken applies to result of the each attempt, including the first.
// Whereas the RetryToken applies to the result of subsequent attempts.
//
// Deprecated: use GetAttemptToken instead.
func (s *Standard) GetInitialToken() func(error) error {
return releaseToken(s.noRetryIncrement).release
}
func (s *Standard) noRetryIncrement() error {
return s.options.RateLimiter.AddTokens(s.options.NoRetryIncrement)
}
// GetRetryToken attempts to deduct the retry cost from the retry token pool.
// Returning the token release function, or error.
func (s *Standard) GetRetryToken(ctx context.Context, opErr error) (func(error) error, error) {
cost := s.options.RetryCost
if s.timeout.IsErrorTimeout(opErr).Bool() {
cost = s.options.RetryTimeoutCost
}
fn, err := s.options.RateLimiter.GetToken(ctx, cost)
if err != nil {
return nil, fmt.Errorf("failed to get rate limit token, %w", err)
}
return releaseToken(fn).release, nil
}
func nopRelease(error) error { return nil }
type releaseToken func() error
func (f releaseToken) release(err error) error {
if err != nil {
return nil
}
return f()
}

View File

@@ -0,0 +1,60 @@
package retry
import (
"errors"
"github.com/aws/aws-sdk-go-v2/aws"
)
// IsErrorThrottle provides the interface of an implementation to determine if
// a error response from an operation is a throttling error.
type IsErrorThrottle interface {
IsErrorThrottle(error) aws.Ternary
}
// IsErrorThrottles is a collection of checks to determine of the error a
// throttle error. Iterates through the checks and returns the state of
// throttle if any check returns something other than unknown.
type IsErrorThrottles []IsErrorThrottle
// IsErrorThrottle returns if the error is a throttle error if any of the
// checks in the list return a value other than unknown.
func (r IsErrorThrottles) IsErrorThrottle(err error) aws.Ternary {
for _, re := range r {
if v := re.IsErrorThrottle(err); v != aws.UnknownTernary {
return v
}
}
return aws.UnknownTernary
}
// IsErrorThrottleFunc wraps a function with the IsErrorThrottle interface.
type IsErrorThrottleFunc func(error) aws.Ternary
// IsErrorThrottle returns if the error is a throttle error.
func (fn IsErrorThrottleFunc) IsErrorThrottle(err error) aws.Ternary {
return fn(err)
}
// ThrottleErrorCode determines if an attempt should be retried based on the
// API error code.
type ThrottleErrorCode struct {
Codes map[string]struct{}
}
// IsErrorThrottle return if the error is a throttle error based on the error
// codes. Returns unknown if the error doesn't have a code or it is unknown.
func (r ThrottleErrorCode) IsErrorThrottle(err error) aws.Ternary {
var v interface{ ErrorCode() string }
if !errors.As(err, &v) {
return aws.UnknownTernary
}
_, ok := r.Codes[v.ErrorCode()]
if !ok {
return aws.UnknownTernary
}
return aws.TrueTernary
}

View File

@@ -0,0 +1,52 @@
package retry
import (
"errors"
"github.com/aws/aws-sdk-go-v2/aws"
)
// IsErrorTimeout provides the interface of an implementation to determine if
// a error matches.
type IsErrorTimeout interface {
IsErrorTimeout(err error) aws.Ternary
}
// IsErrorTimeouts is a collection of checks to determine of the error is
// retryable. Iterates through the checks and returns the state of retryable
// if any check returns something other than unknown.
type IsErrorTimeouts []IsErrorTimeout
// IsErrorTimeout returns if the error is retryable if any of the checks in
// the list return a value other than unknown.
func (ts IsErrorTimeouts) IsErrorTimeout(err error) aws.Ternary {
for _, t := range ts {
if v := t.IsErrorTimeout(err); v != aws.UnknownTernary {
return v
}
}
return aws.UnknownTernary
}
// IsErrorTimeoutFunc wraps a function with the IsErrorTimeout interface.
type IsErrorTimeoutFunc func(error) aws.Ternary
// IsErrorTimeout returns if the error is retryable.
func (fn IsErrorTimeoutFunc) IsErrorTimeout(err error) aws.Ternary {
return fn(err)
}
// TimeouterError provides the IsErrorTimeout implementation for determining if
// an error is a timeout based on type with the Timeout method.
type TimeouterError struct{}
// IsErrorTimeout returns if the error is a timeout error.
func (t TimeouterError) IsErrorTimeout(err error) aws.Ternary {
var v interface{ Timeout() bool }
if !errors.As(err, &v) {
return aws.UnknownTernary
}
return aws.BoolTernary(v.Timeout())
}