From 17c7456b6f4763ed6780e84579d71b4127cf9205 Mon Sep 17 00:00:00 2001 From: Fern Support Date: Tue, 25 Aug 2026 11:25:11 -0400 Subject: [PATCH] fix(go): return the response when the deadline is shorter than the retry backoff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a retryable response arrived and the request context's deadline would elapse before the computed backoff finished, the retrier slept anyway and the caller received `context deadline exceeded` — discarding the response that explained the failure. With `Retry-After` honored up to `maxRetryDelay` (60s), a single 429 could turn into an opaque timeout. The retrier now compares the remaining deadline against the delay and returns the response instead of sleeping, so the caller sees the actual 429 (with its headers and body). An explicit `cancel()` during the backoff still returns the context's error, which is the case `sleepWithContext` exists for. `TestRetryWaitIsInterruptedByContext` previously asserted `context.DeadlineExceeded` for a short-deadline case that this change now resolves to a 429, so it is rewritten to cover cancellation (its actual subject) and two tests are added for the deadline-vs-backoff comparison. Reported by name.com as the optional follow-up to #17498. Co-Authored-By: Claude --- .../go-v2/base/src/asIs/internal/retrier.go_ | 12 ++- .../base/src/asIs/internal/retrier_test.go_ | 90 ++++++++++++++++++- ...nse-when-deadline-shorter-than-backoff.yml | 10 +++ seed/go-sdk/accept-header/internal/retrier.go | 12 ++- .../accept-header/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/alias-extends/internal/retrier.go | 12 ++- .../alias-extends/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/alias/internal/retrier.go | 12 ++- seed/go-sdk/alias/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/allof-inline/internal/retrier.go | 12 ++- .../allof-inline/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/allof/internal/retrier.go | 12 ++- seed/go-sdk/allof/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/any-auth/internal/retrier.go | 12 ++- seed/go-sdk/any-auth/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../api-wide-base-path/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/audiences/internal/retrier.go | 12 ++- .../go-sdk/audiences/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../wire-tests/internal/retrier.go | 12 ++- .../wire-tests/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/basic-auth/internal/retrier.go | 12 ++- .../basic-auth/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../go-sdk/bytes-download/internal/retrier.go | 12 ++- .../bytes-download/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/bytes-upload/internal/retrier.go | 12 ++- .../bytes-upload/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../circular-references/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../client-side-params/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/content-type/internal/retrier.go | 12 ++- .../content-type/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/empty-clients/internal/retrier.go | 12 ++- .../empty-clients/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/enum/internal/retrier.go | 12 ++- seed/go-sdk/enum/internal/retrier_test.go | 90 ++++++++++++++++++- .../go-sdk/error-property/internal/retrier.go | 12 ++- .../error-property/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/errors/internal/retrier.go | 12 ++- seed/go-sdk/errors/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../examples/client-name/internal/retrier.go | 12 ++- .../client-name/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../exported-client-name/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../getters-pass-by-value/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../pleaseinhere/internal/retrier.go | 12 ++- .../pleaseinhere/internal/retrier_test.go | 90 ++++++++++++++++++- .../readme-config/internal/retrier.go | 12 ++- .../readme-config/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/examples/v0/internal/retrier.go | 12 ++- .../examples/v0/internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/extends/internal/retrier.go | 12 ++- seed/go-sdk/extends/internal/retrier_test.go | 90 ++++++++++++++++++- .../extra-properties/internal/retrier.go | 12 ++- .../extra-properties/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/file-download/internal/retrier.go | 12 ++- .../file-download/internal/retrier_test.go | 90 ++++++++++++++++++- .../file-upload-openapi/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../package-name/internal/retrier.go | 12 ++- .../package-name/internal/retrier_test.go | 90 ++++++++++++++++++- .../go-sdk/file-upload/v0/internal/retrier.go | 12 ++- .../file-upload/v0/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/folders/internal/retrier.go | 12 ++- seed/go-sdk/folders/internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../go-content-type/internal/retrier.go | 12 ++- .../go-content-type/internal/retrier_test.go | 90 ++++++++++++++++++- .../go-date-containers/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../go-global-headers/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../go-nullable-date-ref/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/header-auth/internal/retrier.go | 12 ++- .../header-auth/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/http-head/internal/retrier.go | 12 ++- .../go-sdk/http-head/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../idempotency-headers/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../way/in/here/please/internal/retrier.go | 12 ++- .../in/here/please/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../imdb/no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../omit-fern-headers/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../inhereplease/internal/retrier.go | 12 ++- .../inhereplease/internal/retrier_test.go | 90 ++++++++++++++++++- .../with-wiremock-tests/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/license/internal/retrier.go | 12 ++- seed/go-sdk/license/internal/retrier_test.go | 90 ++++++++++++++++++- .../literal-user-agent/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/literal/internal/retrier.go | 12 ++- seed/go-sdk/literal/internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../default-values/internal/retrier.go | 12 ++- .../default-values/internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../mixed-file-directory/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../multi-line-docs/internal/retrier.go | 12 ++- .../multi-line-docs/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../multi-url-environment/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-content-response/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../go-sdk/no-environment/internal/retrier.go | 12 ++- .../no-environment/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/no-retries/internal/retrier.go | 12 ++- .../no-retries/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/null-type/internal/retrier.go | 12 ++- .../go-sdk/null-type/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../nullable-optional/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/nullable/internal/retrier.go | 12 ++- seed/go-sdk/nullable/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/oauth-pkce/internal/retrier.go | 12 ++- .../oauth-pkce/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/object/internal/retrier.go | 12 ++- seed/go-sdk/object/internal/retrier_test.go | 90 ++++++++++++++++++- .../objects-with-imports/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../openapi-subtitle/internal/retrier.go | 12 ++- .../openapi-subtitle/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/optional/internal/retrier.go | 12 ++- seed/go-sdk/optional/internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../pagination-custom/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../pagination-uri-path/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/pagination/internal/retrier.go | 12 ++- .../pagination/internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../package-name/internal/retrier.go | 12 ++- .../package-name/internal/retrier_test.go | 90 ++++++++++++++++++- .../path-parameters/v0/internal/retrier.go | 12 ++- .../v0/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/plain-text/internal/retrier.go | 12 ++- .../plain-text/internal/retrier_test.go | 90 ++++++++++++++++++- .../property-access/internal/retrier.go | 12 ++- .../property-access/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/public-object/internal/retrier.go | 12 ++- .../public-object/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../query-parameters/internal/retrier.go | 12 ++- .../query-parameters/internal/retrier_test.go | 90 ++++++++++++++++++- .../required-nullable/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../reserved-keywords/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../response-property/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../with-wire-tests/internal/retrier.go | 12 ++- .../with-wire-tests/internal/retrier_test.go | 90 ++++++++++++++++++- .../with-wire-tests/internal/retrier.go | 12 ++- .../with-wire-tests/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../with-wire-tests/internal/retrier.go | 12 ++- .../with-wire-tests/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/simple-api/internal/retrier.go | 12 ++- .../simple-api/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/simple-fhir/internal/retrier.go | 12 ++- .../simple-fhir/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../streaming-parameter/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/streaming/internal/retrier.go | 12 ++- .../go-sdk/streaming/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/trace/internal/retrier.go | 12 ++- seed/go-sdk/trace/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../v0/internal/retrier.go | 12 ++- .../v0/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../unions/package-name/internal/retrier.go | 12 ++- .../package-name/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/unions/v0/internal/retrier.go | 12 ++- .../go-sdk/unions/v0/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/unknown/internal/retrier.go | 12 ++- seed/go-sdk/unknown/internal/retrier_test.go | 90 ++++++++++++++++++- .../url-form-encoded/internal/retrier.go | 12 ++- .../url-form-encoded/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/validation/internal/retrier.go | 12 ++- .../validation/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/variables/internal/retrier.go | 12 ++- .../go-sdk/variables/internal/retrier_test.go | 90 ++++++++++++++++++- .../version-no-default/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/version/internal/retrier.go | 12 ++- seed/go-sdk/version/internal/retrier_test.go | 90 ++++++++++++++++++- .../webhook-audience/internal/retrier.go | 12 ++- .../webhook-audience/internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/webhooks/internal/retrier.go | 12 ++- seed/go-sdk/webhooks/internal/retrier_test.go | 90 ++++++++++++++++++- .../websocket-bearer-auth/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../websocket-multi-url/internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- seed/go-sdk/websocket/internal/retrier.go | 12 ++- .../go-sdk/websocket/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- .../no-custom-config/internal/retrier.go | 12 ++- .../no-custom-config/internal/retrier_test.go | 90 ++++++++++++++++++- .../internal/retrier.go | 12 ++- .../internal/retrier_test.go | 90 ++++++++++++++++++- 369 files changed, 17858 insertions(+), 920 deletions(-) create mode 100644 generators/go/sdk/changes/unreleased/return-response-when-deadline-shorter-than-backoff.yml diff --git a/generators/go-v2/base/src/asIs/internal/retrier.go_ b/generators/go-v2/base/src/asIs/internal/retrier.go_ index 3f33e7df0fa7..571a8da31274 100644 --- a/generators/go-v2/base/src/asIs/internal/retrier.go_ +++ b/generators/go-v2/base/src/asIs/internal/retrier.go_ @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/generators/go-v2/base/src/asIs/internal/retrier_test.go_ b/generators/go-v2/base/src/asIs/internal/retrier_test.go_ index e3b19829ebef..a9bc985b40c4 100644 --- a/generators/go-v2/base/src/asIs/internal/retrier_test.go_ +++ b/generators/go-v2/base/src/asIs/internal/retrier_test.go_ @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/generators/go/sdk/changes/unreleased/return-response-when-deadline-shorter-than-backoff.yml b/generators/go/sdk/changes/unreleased/return-response-when-deadline-shorter-than-backoff.yml new file mode 100644 index 000000000000..31c12312b095 --- /dev/null +++ b/generators/go/sdk/changes/unreleased/return-response-when-deadline-shorter-than-backoff.yml @@ -0,0 +1,10 @@ +# yaml-language-server: $schema=../../../../../fern-changes-yml.schema.json + +- summary: | + When a request's context deadline would elapse before the retry backoff + completes, the retrier now returns the response instead of sleeping through + the remaining deadline. Callers see why the request actually failed (for + example a `429` with its `Retry-After` and body) rather than a + `context deadline exceeded` error. An explicit `cancel()` during the + backoff still returns the context's error. + type: fix diff --git a/seed/go-sdk/accept-header/internal/retrier.go b/seed/go-sdk/accept-header/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/accept-header/internal/retrier.go +++ b/seed/go-sdk/accept-header/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/accept-header/internal/retrier_test.go b/seed/go-sdk/accept-header/internal/retrier_test.go index cd5e9fe33260..d55d977088a9 100644 --- a/seed/go-sdk/accept-header/internal/retrier_test.go +++ b/seed/go-sdk/accept-header/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/alias-extends/internal/retrier.go b/seed/go-sdk/alias-extends/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/alias-extends/internal/retrier.go +++ b/seed/go-sdk/alias-extends/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/alias-extends/internal/retrier_test.go b/seed/go-sdk/alias-extends/internal/retrier_test.go index 56d70b69a384..50ad0eda81fa 100644 --- a/seed/go-sdk/alias-extends/internal/retrier_test.go +++ b/seed/go-sdk/alias-extends/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/alias/internal/retrier.go b/seed/go-sdk/alias/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/alias/internal/retrier.go +++ b/seed/go-sdk/alias/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/alias/internal/retrier_test.go b/seed/go-sdk/alias/internal/retrier_test.go index 2262bf46f708..e1ab78d0ada1 100644 --- a/seed/go-sdk/alias/internal/retrier_test.go +++ b/seed/go-sdk/alias/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/allof-inline/internal/retrier.go b/seed/go-sdk/allof-inline/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/allof-inline/internal/retrier.go +++ b/seed/go-sdk/allof-inline/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/allof-inline/internal/retrier_test.go b/seed/go-sdk/allof-inline/internal/retrier_test.go index 189c82bff4d2..51b0f1c12d62 100644 --- a/seed/go-sdk/allof-inline/internal/retrier_test.go +++ b/seed/go-sdk/allof-inline/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/allof/internal/retrier.go b/seed/go-sdk/allof/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/allof/internal/retrier.go +++ b/seed/go-sdk/allof/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/allof/internal/retrier_test.go b/seed/go-sdk/allof/internal/retrier_test.go index 7aa6938d813b..aaf5745f9df4 100644 --- a/seed/go-sdk/allof/internal/retrier_test.go +++ b/seed/go-sdk/allof/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/any-auth/internal/retrier.go b/seed/go-sdk/any-auth/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/any-auth/internal/retrier.go +++ b/seed/go-sdk/any-auth/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/any-auth/internal/retrier_test.go b/seed/go-sdk/any-auth/internal/retrier_test.go index b77161ed33c0..2a2f7c5d47c1 100644 --- a/seed/go-sdk/any-auth/internal/retrier_test.go +++ b/seed/go-sdk/any-auth/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/api-wide-base-path-with-default/internal/retrier.go b/seed/go-sdk/api-wide-base-path-with-default/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/api-wide-base-path-with-default/internal/retrier.go +++ b/seed/go-sdk/api-wide-base-path-with-default/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/api-wide-base-path-with-default/internal/retrier_test.go b/seed/go-sdk/api-wide-base-path-with-default/internal/retrier_test.go index 12961291aaa2..43938bc0b192 100644 --- a/seed/go-sdk/api-wide-base-path-with-default/internal/retrier_test.go +++ b/seed/go-sdk/api-wide-base-path-with-default/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/api-wide-base-path/internal/retrier.go b/seed/go-sdk/api-wide-base-path/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/api-wide-base-path/internal/retrier.go +++ b/seed/go-sdk/api-wide-base-path/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/api-wide-base-path/internal/retrier_test.go b/seed/go-sdk/api-wide-base-path/internal/retrier_test.go index 6dd81534c910..b7b63265f380 100644 --- a/seed/go-sdk/api-wide-base-path/internal/retrier_test.go +++ b/seed/go-sdk/api-wide-base-path/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/audiences/internal/retrier.go b/seed/go-sdk/audiences/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/audiences/internal/retrier.go +++ b/seed/go-sdk/audiences/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/audiences/internal/retrier_test.go b/seed/go-sdk/audiences/internal/retrier_test.go index 38c53869a39b..557b894840c7 100644 --- a/seed/go-sdk/audiences/internal/retrier_test.go +++ b/seed/go-sdk/audiences/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/basic-auth-environment-variables/internal/retrier.go b/seed/go-sdk/basic-auth-environment-variables/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/basic-auth-environment-variables/internal/retrier.go +++ b/seed/go-sdk/basic-auth-environment-variables/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/basic-auth-environment-variables/internal/retrier_test.go b/seed/go-sdk/basic-auth-environment-variables/internal/retrier_test.go index 89d131c9b1e8..436e92d8b8b9 100644 --- a/seed/go-sdk/basic-auth-environment-variables/internal/retrier_test.go +++ b/seed/go-sdk/basic-auth-environment-variables/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/basic-auth-pw-omitted/wire-tests/internal/retrier.go b/seed/go-sdk/basic-auth-pw-omitted/wire-tests/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/basic-auth-pw-omitted/wire-tests/internal/retrier.go +++ b/seed/go-sdk/basic-auth-pw-omitted/wire-tests/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/basic-auth-pw-omitted/wire-tests/internal/retrier_test.go b/seed/go-sdk/basic-auth-pw-omitted/wire-tests/internal/retrier_test.go index 0ae146270b57..d4d17e4176c2 100644 --- a/seed/go-sdk/basic-auth-pw-omitted/wire-tests/internal/retrier_test.go +++ b/seed/go-sdk/basic-auth-pw-omitted/wire-tests/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/basic-auth/internal/retrier.go b/seed/go-sdk/basic-auth/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/basic-auth/internal/retrier.go +++ b/seed/go-sdk/basic-auth/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/basic-auth/internal/retrier_test.go b/seed/go-sdk/basic-auth/internal/retrier_test.go index 9f9bc5b63a84..463d18a60492 100644 --- a/seed/go-sdk/basic-auth/internal/retrier_test.go +++ b/seed/go-sdk/basic-auth/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/bearer-token-environment-variable/internal/retrier.go b/seed/go-sdk/bearer-token-environment-variable/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/bearer-token-environment-variable/internal/retrier.go +++ b/seed/go-sdk/bearer-token-environment-variable/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/bearer-token-environment-variable/internal/retrier_test.go b/seed/go-sdk/bearer-token-environment-variable/internal/retrier_test.go index 927db22068bf..378415cd03b0 100644 --- a/seed/go-sdk/bearer-token-environment-variable/internal/retrier_test.go +++ b/seed/go-sdk/bearer-token-environment-variable/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/bytes-download/internal/retrier.go b/seed/go-sdk/bytes-download/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/bytes-download/internal/retrier.go +++ b/seed/go-sdk/bytes-download/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/bytes-download/internal/retrier_test.go b/seed/go-sdk/bytes-download/internal/retrier_test.go index f7d2a1bbf099..642719d5e94d 100644 --- a/seed/go-sdk/bytes-download/internal/retrier_test.go +++ b/seed/go-sdk/bytes-download/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/bytes-upload/internal/retrier.go b/seed/go-sdk/bytes-upload/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/bytes-upload/internal/retrier.go +++ b/seed/go-sdk/bytes-upload/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/bytes-upload/internal/retrier_test.go b/seed/go-sdk/bytes-upload/internal/retrier_test.go index 3b76cbd9b60c..ba5624ee5283 100644 --- a/seed/go-sdk/bytes-upload/internal/retrier_test.go +++ b/seed/go-sdk/bytes-upload/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/circular-references-advanced/internal/retrier.go b/seed/go-sdk/circular-references-advanced/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/circular-references-advanced/internal/retrier.go +++ b/seed/go-sdk/circular-references-advanced/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/circular-references-advanced/internal/retrier_test.go b/seed/go-sdk/circular-references-advanced/internal/retrier_test.go index bdfd0285587a..a53327512027 100644 --- a/seed/go-sdk/circular-references-advanced/internal/retrier_test.go +++ b/seed/go-sdk/circular-references-advanced/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/circular-references-extends/internal/retrier.go b/seed/go-sdk/circular-references-extends/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/circular-references-extends/internal/retrier.go +++ b/seed/go-sdk/circular-references-extends/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/circular-references-extends/internal/retrier_test.go b/seed/go-sdk/circular-references-extends/internal/retrier_test.go index bf04f64407b3..6249f57f2bab 100644 --- a/seed/go-sdk/circular-references-extends/internal/retrier_test.go +++ b/seed/go-sdk/circular-references-extends/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/circular-references/internal/retrier.go b/seed/go-sdk/circular-references/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/circular-references/internal/retrier.go +++ b/seed/go-sdk/circular-references/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/circular-references/internal/retrier_test.go b/seed/go-sdk/circular-references/internal/retrier_test.go index 86a7bde4108a..2e55270997c4 100644 --- a/seed/go-sdk/circular-references/internal/retrier_test.go +++ b/seed/go-sdk/circular-references/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/client-side-params/internal/retrier.go b/seed/go-sdk/client-side-params/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/client-side-params/internal/retrier.go +++ b/seed/go-sdk/client-side-params/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/client-side-params/internal/retrier_test.go b/seed/go-sdk/client-side-params/internal/retrier_test.go index 6e39eff7e551..5b8ccecd3ffa 100644 --- a/seed/go-sdk/client-side-params/internal/retrier_test.go +++ b/seed/go-sdk/client-side-params/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/content-type/internal/retrier.go b/seed/go-sdk/content-type/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/content-type/internal/retrier.go +++ b/seed/go-sdk/content-type/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/content-type/internal/retrier_test.go b/seed/go-sdk/content-type/internal/retrier_test.go index 64b8b584f060..0bbbd604ffb2 100644 --- a/seed/go-sdk/content-type/internal/retrier_test.go +++ b/seed/go-sdk/content-type/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/cross-package-type-names/internal/retrier.go b/seed/go-sdk/cross-package-type-names/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/cross-package-type-names/internal/retrier.go +++ b/seed/go-sdk/cross-package-type-names/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/cross-package-type-names/internal/retrier_test.go b/seed/go-sdk/cross-package-type-names/internal/retrier_test.go index 8afee2a11b22..e8ee02603cca 100644 --- a/seed/go-sdk/cross-package-type-names/internal/retrier_test.go +++ b/seed/go-sdk/cross-package-type-names/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/discriminated-union-with-nested-oneof/internal/retrier.go b/seed/go-sdk/discriminated-union-with-nested-oneof/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/discriminated-union-with-nested-oneof/internal/retrier.go +++ b/seed/go-sdk/discriminated-union-with-nested-oneof/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/discriminated-union-with-nested-oneof/internal/retrier_test.go b/seed/go-sdk/discriminated-union-with-nested-oneof/internal/retrier_test.go index 5c87a249b386..0dacb3868636 100644 --- a/seed/go-sdk/discriminated-union-with-nested-oneof/internal/retrier_test.go +++ b/seed/go-sdk/discriminated-union-with-nested-oneof/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/dollar-string-examples/internal/retrier.go b/seed/go-sdk/dollar-string-examples/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/dollar-string-examples/internal/retrier.go +++ b/seed/go-sdk/dollar-string-examples/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/dollar-string-examples/internal/retrier_test.go b/seed/go-sdk/dollar-string-examples/internal/retrier_test.go index f4257d1e512a..985fe1c0001c 100644 --- a/seed/go-sdk/dollar-string-examples/internal/retrier_test.go +++ b/seed/go-sdk/dollar-string-examples/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/empty-clients/internal/retrier.go b/seed/go-sdk/empty-clients/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/empty-clients/internal/retrier.go +++ b/seed/go-sdk/empty-clients/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/empty-clients/internal/retrier_test.go b/seed/go-sdk/empty-clients/internal/retrier_test.go index 7c2c402bae8e..3aa665a20869 100644 --- a/seed/go-sdk/empty-clients/internal/retrier_test.go +++ b/seed/go-sdk/empty-clients/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/endpoint-security-auth/internal/retrier.go b/seed/go-sdk/endpoint-security-auth/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/endpoint-security-auth/internal/retrier.go +++ b/seed/go-sdk/endpoint-security-auth/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/endpoint-security-auth/internal/retrier_test.go b/seed/go-sdk/endpoint-security-auth/internal/retrier_test.go index 8a6aafd41de4..1803513a73c6 100644 --- a/seed/go-sdk/endpoint-security-auth/internal/retrier_test.go +++ b/seed/go-sdk/endpoint-security-auth/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/enum/internal/retrier.go b/seed/go-sdk/enum/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/enum/internal/retrier.go +++ b/seed/go-sdk/enum/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/enum/internal/retrier_test.go b/seed/go-sdk/enum/internal/retrier_test.go index b2c86a9a4d48..f17cfb993a34 100644 --- a/seed/go-sdk/enum/internal/retrier_test.go +++ b/seed/go-sdk/enum/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/error-property/internal/retrier.go b/seed/go-sdk/error-property/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/error-property/internal/retrier.go +++ b/seed/go-sdk/error-property/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/error-property/internal/retrier_test.go b/seed/go-sdk/error-property/internal/retrier_test.go index d2b4b2c83d82..885a7db266f1 100644 --- a/seed/go-sdk/error-property/internal/retrier_test.go +++ b/seed/go-sdk/error-property/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/errors/internal/retrier.go b/seed/go-sdk/errors/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/errors/internal/retrier.go +++ b/seed/go-sdk/errors/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/errors/internal/retrier_test.go b/seed/go-sdk/errors/internal/retrier_test.go index 9d56faad7291..202be552ad98 100644 --- a/seed/go-sdk/errors/internal/retrier_test.go +++ b/seed/go-sdk/errors/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/examples/always-send-required-properties/internal/retrier.go b/seed/go-sdk/examples/always-send-required-properties/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/examples/always-send-required-properties/internal/retrier.go +++ b/seed/go-sdk/examples/always-send-required-properties/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/examples/always-send-required-properties/internal/retrier_test.go b/seed/go-sdk/examples/always-send-required-properties/internal/retrier_test.go index d51394518f92..d1a763c763b8 100644 --- a/seed/go-sdk/examples/always-send-required-properties/internal/retrier_test.go +++ b/seed/go-sdk/examples/always-send-required-properties/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/examples/client-name-with-custom-constructor-name/internal/retrier.go b/seed/go-sdk/examples/client-name-with-custom-constructor-name/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/examples/client-name-with-custom-constructor-name/internal/retrier.go +++ b/seed/go-sdk/examples/client-name-with-custom-constructor-name/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/examples/client-name-with-custom-constructor-name/internal/retrier_test.go b/seed/go-sdk/examples/client-name-with-custom-constructor-name/internal/retrier_test.go index d51394518f92..d1a763c763b8 100644 --- a/seed/go-sdk/examples/client-name-with-custom-constructor-name/internal/retrier_test.go +++ b/seed/go-sdk/examples/client-name-with-custom-constructor-name/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/examples/client-name/internal/retrier.go b/seed/go-sdk/examples/client-name/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/examples/client-name/internal/retrier.go +++ b/seed/go-sdk/examples/client-name/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/examples/client-name/internal/retrier_test.go b/seed/go-sdk/examples/client-name/internal/retrier_test.go index d51394518f92..d1a763c763b8 100644 --- a/seed/go-sdk/examples/client-name/internal/retrier_test.go +++ b/seed/go-sdk/examples/client-name/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/examples/export-all-requests-at-root/internal/retrier.go b/seed/go-sdk/examples/export-all-requests-at-root/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/examples/export-all-requests-at-root/internal/retrier.go +++ b/seed/go-sdk/examples/export-all-requests-at-root/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/examples/export-all-requests-at-root/internal/retrier_test.go b/seed/go-sdk/examples/export-all-requests-at-root/internal/retrier_test.go index d51394518f92..d1a763c763b8 100644 --- a/seed/go-sdk/examples/export-all-requests-at-root/internal/retrier_test.go +++ b/seed/go-sdk/examples/export-all-requests-at-root/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/examples/exported-client-name/internal/retrier.go b/seed/go-sdk/examples/exported-client-name/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/examples/exported-client-name/internal/retrier.go +++ b/seed/go-sdk/examples/exported-client-name/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/examples/exported-client-name/internal/retrier_test.go b/seed/go-sdk/examples/exported-client-name/internal/retrier_test.go index d51394518f92..d1a763c763b8 100644 --- a/seed/go-sdk/examples/exported-client-name/internal/retrier_test.go +++ b/seed/go-sdk/examples/exported-client-name/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/examples/getters-pass-by-value/internal/retrier.go b/seed/go-sdk/examples/getters-pass-by-value/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/examples/getters-pass-by-value/internal/retrier.go +++ b/seed/go-sdk/examples/getters-pass-by-value/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/examples/getters-pass-by-value/internal/retrier_test.go b/seed/go-sdk/examples/getters-pass-by-value/internal/retrier_test.go index d51394518f92..d1a763c763b8 100644 --- a/seed/go-sdk/examples/getters-pass-by-value/internal/retrier_test.go +++ b/seed/go-sdk/examples/getters-pass-by-value/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/examples/no-custom-config/internal/retrier.go b/seed/go-sdk/examples/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/examples/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/examples/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/examples/no-custom-config/internal/retrier_test.go b/seed/go-sdk/examples/no-custom-config/internal/retrier_test.go index d51394518f92..d1a763c763b8 100644 --- a/seed/go-sdk/examples/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/examples/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/examples/package-path/pleaseinhere/internal/retrier.go b/seed/go-sdk/examples/package-path/pleaseinhere/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/examples/package-path/pleaseinhere/internal/retrier.go +++ b/seed/go-sdk/examples/package-path/pleaseinhere/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/examples/package-path/pleaseinhere/internal/retrier_test.go b/seed/go-sdk/examples/package-path/pleaseinhere/internal/retrier_test.go index 533b5694339c..a4628dbefbb3 100644 --- a/seed/go-sdk/examples/package-path/pleaseinhere/internal/retrier_test.go +++ b/seed/go-sdk/examples/package-path/pleaseinhere/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/examples/readme-config/internal/retrier.go b/seed/go-sdk/examples/readme-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/examples/readme-config/internal/retrier.go +++ b/seed/go-sdk/examples/readme-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/examples/readme-config/internal/retrier_test.go b/seed/go-sdk/examples/readme-config/internal/retrier_test.go index d51394518f92..d1a763c763b8 100644 --- a/seed/go-sdk/examples/readme-config/internal/retrier_test.go +++ b/seed/go-sdk/examples/readme-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/examples/v0/internal/retrier.go b/seed/go-sdk/examples/v0/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/examples/v0/internal/retrier.go +++ b/seed/go-sdk/examples/v0/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/examples/v0/internal/retrier_test.go b/seed/go-sdk/examples/v0/internal/retrier_test.go index d51394518f92..d1a763c763b8 100644 --- a/seed/go-sdk/examples/v0/internal/retrier_test.go +++ b/seed/go-sdk/examples/v0/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/exhaustive/no-custom-config/internal/retrier.go b/seed/go-sdk/exhaustive/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/exhaustive/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/exhaustive/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/exhaustive/no-custom-config/internal/retrier_test.go b/seed/go-sdk/exhaustive/no-custom-config/internal/retrier_test.go index 19e78d0ad107..8d364d2ebae2 100644 --- a/seed/go-sdk/exhaustive/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/exhaustive/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/exhaustive/omit-empty-request-wrappers/internal/retrier.go b/seed/go-sdk/exhaustive/omit-empty-request-wrappers/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/exhaustive/omit-empty-request-wrappers/internal/retrier.go +++ b/seed/go-sdk/exhaustive/omit-empty-request-wrappers/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/exhaustive/omit-empty-request-wrappers/internal/retrier_test.go b/seed/go-sdk/exhaustive/omit-empty-request-wrappers/internal/retrier_test.go index 19e78d0ad107..8d364d2ebae2 100644 --- a/seed/go-sdk/exhaustive/omit-empty-request-wrappers/internal/retrier_test.go +++ b/seed/go-sdk/exhaustive/omit-empty-request-wrappers/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/extends/internal/retrier.go b/seed/go-sdk/extends/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/extends/internal/retrier.go +++ b/seed/go-sdk/extends/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/extends/internal/retrier_test.go b/seed/go-sdk/extends/internal/retrier_test.go index a0f1dfcf477a..6620d9545c66 100644 --- a/seed/go-sdk/extends/internal/retrier_test.go +++ b/seed/go-sdk/extends/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/extra-properties/internal/retrier.go b/seed/go-sdk/extra-properties/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/extra-properties/internal/retrier.go +++ b/seed/go-sdk/extra-properties/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/extra-properties/internal/retrier_test.go b/seed/go-sdk/extra-properties/internal/retrier_test.go index 9fc446ee3cbe..94fd2add9d26 100644 --- a/seed/go-sdk/extra-properties/internal/retrier_test.go +++ b/seed/go-sdk/extra-properties/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/file-download/internal/retrier.go b/seed/go-sdk/file-download/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/file-download/internal/retrier.go +++ b/seed/go-sdk/file-download/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/file-download/internal/retrier_test.go b/seed/go-sdk/file-download/internal/retrier_test.go index b8e61c4e87d3..53b2f2aef1ad 100644 --- a/seed/go-sdk/file-download/internal/retrier_test.go +++ b/seed/go-sdk/file-download/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/file-upload-openapi/internal/retrier.go b/seed/go-sdk/file-upload-openapi/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/file-upload-openapi/internal/retrier.go +++ b/seed/go-sdk/file-upload-openapi/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/file-upload-openapi/internal/retrier_test.go b/seed/go-sdk/file-upload-openapi/internal/retrier_test.go index cf62c7d230bd..0f37db849842 100644 --- a/seed/go-sdk/file-upload-openapi/internal/retrier_test.go +++ b/seed/go-sdk/file-upload-openapi/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/file-upload/no-custom-config/internal/retrier.go b/seed/go-sdk/file-upload/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/file-upload/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/file-upload/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/file-upload/no-custom-config/internal/retrier_test.go b/seed/go-sdk/file-upload/no-custom-config/internal/retrier_test.go index dfb890c75995..3efdafc646dc 100644 --- a/seed/go-sdk/file-upload/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/file-upload/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/file-upload/package-name/internal/retrier.go b/seed/go-sdk/file-upload/package-name/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/file-upload/package-name/internal/retrier.go +++ b/seed/go-sdk/file-upload/package-name/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/file-upload/package-name/internal/retrier_test.go b/seed/go-sdk/file-upload/package-name/internal/retrier_test.go index 0381e575c4b8..758a27e902d6 100644 --- a/seed/go-sdk/file-upload/package-name/internal/retrier_test.go +++ b/seed/go-sdk/file-upload/package-name/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/file-upload/v0/internal/retrier.go b/seed/go-sdk/file-upload/v0/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/file-upload/v0/internal/retrier.go +++ b/seed/go-sdk/file-upload/v0/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/file-upload/v0/internal/retrier_test.go b/seed/go-sdk/file-upload/v0/internal/retrier_test.go index dfb890c75995..3efdafc646dc 100644 --- a/seed/go-sdk/file-upload/v0/internal/retrier_test.go +++ b/seed/go-sdk/file-upload/v0/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/folders/internal/retrier.go b/seed/go-sdk/folders/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/folders/internal/retrier.go +++ b/seed/go-sdk/folders/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/folders/internal/retrier_test.go b/seed/go-sdk/folders/internal/retrier_test.go index 41530104b364..3c00fca59b77 100644 --- a/seed/go-sdk/folders/internal/retrier_test.go +++ b/seed/go-sdk/folders/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-bytes-request/no-custom-config/internal/retrier.go b/seed/go-sdk/go-bytes-request/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-bytes-request/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/go-bytes-request/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-bytes-request/no-custom-config/internal/retrier_test.go b/seed/go-sdk/go-bytes-request/no-custom-config/internal/retrier_test.go index 604065f9cfd5..64e0ae773ae7 100644 --- a/seed/go-sdk/go-bytes-request/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/go-bytes-request/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-bytes-request/use-reader-for-bytes-request/internal/retrier.go b/seed/go-sdk/go-bytes-request/use-reader-for-bytes-request/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-bytes-request/use-reader-for-bytes-request/internal/retrier.go +++ b/seed/go-sdk/go-bytes-request/use-reader-for-bytes-request/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-bytes-request/use-reader-for-bytes-request/internal/retrier_test.go b/seed/go-sdk/go-bytes-request/use-reader-for-bytes-request/internal/retrier_test.go index 604065f9cfd5..64e0ae773ae7 100644 --- a/seed/go-sdk/go-bytes-request/use-reader-for-bytes-request/internal/retrier_test.go +++ b/seed/go-sdk/go-bytes-request/use-reader-for-bytes-request/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-content-type/internal/retrier.go b/seed/go-sdk/go-content-type/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-content-type/internal/retrier.go +++ b/seed/go-sdk/go-content-type/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-content-type/internal/retrier_test.go b/seed/go-sdk/go-content-type/internal/retrier_test.go index 85235b05a523..b78e6d32e72c 100644 --- a/seed/go-sdk/go-content-type/internal/retrier_test.go +++ b/seed/go-sdk/go-content-type/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-date-containers/internal/retrier.go b/seed/go-sdk/go-date-containers/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-date-containers/internal/retrier.go +++ b/seed/go-sdk/go-date-containers/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-date-containers/internal/retrier_test.go b/seed/go-sdk/go-date-containers/internal/retrier_test.go index f0e3eebb3f0d..d3af5e24a895 100644 --- a/seed/go-sdk/go-date-containers/internal/retrier_test.go +++ b/seed/go-sdk/go-date-containers/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-deterministic-ordering/internal/retrier.go b/seed/go-sdk/go-deterministic-ordering/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-deterministic-ordering/internal/retrier.go +++ b/seed/go-sdk/go-deterministic-ordering/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-deterministic-ordering/internal/retrier_test.go b/seed/go-sdk/go-deterministic-ordering/internal/retrier_test.go index afed94338b0e..ff6af65ddbb0 100644 --- a/seed/go-sdk/go-deterministic-ordering/internal/retrier_test.go +++ b/seed/go-sdk/go-deterministic-ordering/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-global-headers/internal/retrier.go b/seed/go-sdk/go-global-headers/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-global-headers/internal/retrier.go +++ b/seed/go-sdk/go-global-headers/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-global-headers/internal/retrier_test.go b/seed/go-sdk/go-global-headers/internal/retrier_test.go index 625787f5b434..d1f39ef7d38a 100644 --- a/seed/go-sdk/go-global-headers/internal/retrier_test.go +++ b/seed/go-sdk/go-global-headers/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-multi-env-url-templating/no-custom-config/internal/retrier.go b/seed/go-sdk/go-multi-env-url-templating/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-multi-env-url-templating/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/go-multi-env-url-templating/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-multi-env-url-templating/no-custom-config/internal/retrier_test.go b/seed/go-sdk/go-multi-env-url-templating/no-custom-config/internal/retrier_test.go index 50f1c14468ff..98529b9243e9 100644 --- a/seed/go-sdk/go-multi-env-url-templating/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/go-multi-env-url-templating/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-nested-cross-package/internal/retrier.go b/seed/go-sdk/go-nested-cross-package/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-nested-cross-package/internal/retrier.go +++ b/seed/go-sdk/go-nested-cross-package/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-nested-cross-package/internal/retrier_test.go b/seed/go-sdk/go-nested-cross-package/internal/retrier_test.go index d65d61299842..835fc1ddc5ed 100644 --- a/seed/go-sdk/go-nested-cross-package/internal/retrier_test.go +++ b/seed/go-sdk/go-nested-cross-package/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-nullable-date-ref/internal/retrier.go b/seed/go-sdk/go-nullable-date-ref/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-nullable-date-ref/internal/retrier.go +++ b/seed/go-sdk/go-nullable-date-ref/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-nullable-date-ref/internal/retrier_test.go b/seed/go-sdk/go-nullable-date-ref/internal/retrier_test.go index b1642bb8ac93..034a073c4d4a 100644 --- a/seed/go-sdk/go-nullable-date-ref/internal/retrier_test.go +++ b/seed/go-sdk/go-nullable-date-ref/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-nullable-wire-tests/internal/retrier.go b/seed/go-sdk/go-nullable-wire-tests/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-nullable-wire-tests/internal/retrier.go +++ b/seed/go-sdk/go-nullable-wire-tests/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-nullable-wire-tests/internal/retrier_test.go b/seed/go-sdk/go-nullable-wire-tests/internal/retrier_test.go index fd2fe2db6759..49dbb9ecc636 100644 --- a/seed/go-sdk/go-nullable-wire-tests/internal/retrier_test.go +++ b/seed/go-sdk/go-nullable-wire-tests/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-oauth-token-nullable/no-custom-config/internal/retrier.go b/seed/go-sdk/go-oauth-token-nullable/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-oauth-token-nullable/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/go-oauth-token-nullable/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-oauth-token-nullable/no-custom-config/internal/retrier_test.go b/seed/go-sdk/go-oauth-token-nullable/no-custom-config/internal/retrier_test.go index 194489383f81..9443b556e366 100644 --- a/seed/go-sdk/go-oauth-token-nullable/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/go-oauth-token-nullable/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-oauth-token-optional/no-custom-config/internal/retrier.go b/seed/go-sdk/go-oauth-token-optional/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-oauth-token-optional/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/go-oauth-token-optional/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-oauth-token-optional/no-custom-config/internal/retrier_test.go b/seed/go-sdk/go-oauth-token-optional/no-custom-config/internal/retrier_test.go index e3e1cdbce791..8b68208fd4be 100644 --- a/seed/go-sdk/go-oauth-token-optional/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/go-oauth-token-optional/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-optional-header-env/internal/retrier.go b/seed/go-sdk/go-optional-header-env/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-optional-header-env/internal/retrier.go +++ b/seed/go-sdk/go-optional-header-env/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-optional-header-env/internal/retrier_test.go b/seed/go-sdk/go-optional-header-env/internal/retrier_test.go index 6f3fa76fd9b8..2d07eea95fb9 100644 --- a/seed/go-sdk/go-optional-header-env/internal/retrier_test.go +++ b/seed/go-sdk/go-optional-header-env/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-optional-literal-alias/internal/retrier.go b/seed/go-sdk/go-optional-literal-alias/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-optional-literal-alias/internal/retrier.go +++ b/seed/go-sdk/go-optional-literal-alias/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-optional-literal-alias/internal/retrier_test.go b/seed/go-sdk/go-optional-literal-alias/internal/retrier_test.go index df35ab34820b..728de1f906ec 100644 --- a/seed/go-sdk/go-optional-literal-alias/internal/retrier_test.go +++ b/seed/go-sdk/go-optional-literal-alias/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-optional-request-body/no-custom-config/internal/retrier.go b/seed/go-sdk/go-optional-request-body/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-optional-request-body/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/go-optional-request-body/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-optional-request-body/no-custom-config/internal/retrier_test.go b/seed/go-sdk/go-optional-request-body/no-custom-config/internal/retrier_test.go index db29948c2926..5b1a55153ffa 100644 --- a/seed/go-sdk/go-optional-request-body/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/go-optional-request-body/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-optional-request-body/respect-optional-request-body/internal/retrier.go b/seed/go-sdk/go-optional-request-body/respect-optional-request-body/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-optional-request-body/respect-optional-request-body/internal/retrier.go +++ b/seed/go-sdk/go-optional-request-body/respect-optional-request-body/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-optional-request-body/respect-optional-request-body/internal/retrier_test.go b/seed/go-sdk/go-optional-request-body/respect-optional-request-body/internal/retrier_test.go index db29948c2926..5b1a55153ffa 100644 --- a/seed/go-sdk/go-optional-request-body/respect-optional-request-body/internal/retrier_test.go +++ b/seed/go-sdk/go-optional-request-body/respect-optional-request-body/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-pagination-offset-item-index/internal/retrier.go b/seed/go-sdk/go-pagination-offset-item-index/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-pagination-offset-item-index/internal/retrier.go +++ b/seed/go-sdk/go-pagination-offset-item-index/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-pagination-offset-item-index/internal/retrier_test.go b/seed/go-sdk/go-pagination-offset-item-index/internal/retrier_test.go index 8df644cd0ce8..20e0cf9aae91 100644 --- a/seed/go-sdk/go-pagination-offset-item-index/internal/retrier_test.go +++ b/seed/go-sdk/go-pagination-offset-item-index/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-request-body-pagination/internal/retrier.go b/seed/go-sdk/go-request-body-pagination/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-request-body-pagination/internal/retrier.go +++ b/seed/go-sdk/go-request-body-pagination/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-request-body-pagination/internal/retrier_test.go b/seed/go-sdk/go-request-body-pagination/internal/retrier_test.go index d064cda3a0d3..92a1903d9798 100644 --- a/seed/go-sdk/go-request-body-pagination/internal/retrier_test.go +++ b/seed/go-sdk/go-request-body-pagination/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-subpackage-collision/internal/retrier.go b/seed/go-sdk/go-subpackage-collision/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-subpackage-collision/internal/retrier.go +++ b/seed/go-sdk/go-subpackage-collision/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-subpackage-collision/internal/retrier_test.go b/seed/go-sdk/go-subpackage-collision/internal/retrier_test.go index 311381445478..5fe43f62b852 100644 --- a/seed/go-sdk/go-subpackage-collision/internal/retrier_test.go +++ b/seed/go-sdk/go-subpackage-collision/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-undiscriminated-union-wire-tests/internal/retrier.go b/seed/go-sdk/go-undiscriminated-union-wire-tests/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-undiscriminated-union-wire-tests/internal/retrier.go +++ b/seed/go-sdk/go-undiscriminated-union-wire-tests/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-undiscriminated-union-wire-tests/internal/retrier_test.go b/seed/go-sdk/go-undiscriminated-union-wire-tests/internal/retrier_test.go index 66f1d66159a6..429dc36bcd69 100644 --- a/seed/go-sdk/go-undiscriminated-union-wire-tests/internal/retrier_test.go +++ b/seed/go-sdk/go-undiscriminated-union-wire-tests/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-union-base-properties/dedupe-union-base-properties/internal/retrier.go b/seed/go-sdk/go-union-base-properties/dedupe-union-base-properties/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-union-base-properties/dedupe-union-base-properties/internal/retrier.go +++ b/seed/go-sdk/go-union-base-properties/dedupe-union-base-properties/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-union-base-properties/dedupe-union-base-properties/internal/retrier_test.go b/seed/go-sdk/go-union-base-properties/dedupe-union-base-properties/internal/retrier_test.go index f371607edaae..f06d6379b6c5 100644 --- a/seed/go-sdk/go-union-base-properties/dedupe-union-base-properties/internal/retrier_test.go +++ b/seed/go-sdk/go-union-base-properties/dedupe-union-base-properties/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/go-union-base-properties/no-custom-config/internal/retrier.go b/seed/go-sdk/go-union-base-properties/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/go-union-base-properties/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/go-union-base-properties/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/go-union-base-properties/no-custom-config/internal/retrier_test.go b/seed/go-sdk/go-union-base-properties/no-custom-config/internal/retrier_test.go index f371607edaae..f06d6379b6c5 100644 --- a/seed/go-sdk/go-union-base-properties/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/go-union-base-properties/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/header-auth-environment-variable/internal/retrier.go b/seed/go-sdk/header-auth-environment-variable/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/header-auth-environment-variable/internal/retrier.go +++ b/seed/go-sdk/header-auth-environment-variable/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/header-auth-environment-variable/internal/retrier_test.go b/seed/go-sdk/header-auth-environment-variable/internal/retrier_test.go index 68170fe69b10..dcce44f2e464 100644 --- a/seed/go-sdk/header-auth-environment-variable/internal/retrier_test.go +++ b/seed/go-sdk/header-auth-environment-variable/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/header-auth/internal/retrier.go b/seed/go-sdk/header-auth/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/header-auth/internal/retrier.go +++ b/seed/go-sdk/header-auth/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/header-auth/internal/retrier_test.go b/seed/go-sdk/header-auth/internal/retrier_test.go index b2b13cf5dd8e..290ec216bb62 100644 --- a/seed/go-sdk/header-auth/internal/retrier_test.go +++ b/seed/go-sdk/header-auth/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/http-head/internal/retrier.go b/seed/go-sdk/http-head/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/http-head/internal/retrier.go +++ b/seed/go-sdk/http-head/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/http-head/internal/retrier_test.go b/seed/go-sdk/http-head/internal/retrier_test.go index 319d785a3f72..5e0c45043108 100644 --- a/seed/go-sdk/http-head/internal/retrier_test.go +++ b/seed/go-sdk/http-head/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/idempotency-headers/auto-generate-idempotency-key/internal/retrier.go b/seed/go-sdk/idempotency-headers/auto-generate-idempotency-key/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/idempotency-headers/auto-generate-idempotency-key/internal/retrier.go +++ b/seed/go-sdk/idempotency-headers/auto-generate-idempotency-key/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/idempotency-headers/auto-generate-idempotency-key/internal/retrier_test.go b/seed/go-sdk/idempotency-headers/auto-generate-idempotency-key/internal/retrier_test.go index 5f0fe080884c..b31bf5b55a38 100644 --- a/seed/go-sdk/idempotency-headers/auto-generate-idempotency-key/internal/retrier_test.go +++ b/seed/go-sdk/idempotency-headers/auto-generate-idempotency-key/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/idempotency-headers/internal/retrier.go b/seed/go-sdk/idempotency-headers/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/idempotency-headers/internal/retrier.go +++ b/seed/go-sdk/idempotency-headers/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/idempotency-headers/internal/retrier_test.go b/seed/go-sdk/idempotency-headers/internal/retrier_test.go index 5f0fe080884c..b31bf5b55a38 100644 --- a/seed/go-sdk/idempotency-headers/internal/retrier_test.go +++ b/seed/go-sdk/idempotency-headers/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/imdb/allow-user-agent-app-info/internal/retrier.go b/seed/go-sdk/imdb/allow-user-agent-app-info/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/imdb/allow-user-agent-app-info/internal/retrier.go +++ b/seed/go-sdk/imdb/allow-user-agent-app-info/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/imdb/allow-user-agent-app-info/internal/retrier_test.go b/seed/go-sdk/imdb/allow-user-agent-app-info/internal/retrier_test.go index b9e74b043e11..09b373a2a3b5 100644 --- a/seed/go-sdk/imdb/allow-user-agent-app-info/internal/retrier_test.go +++ b/seed/go-sdk/imdb/allow-user-agent-app-info/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/imdb/deep-package-path/all/the/way/in/here/please/internal/retrier.go b/seed/go-sdk/imdb/deep-package-path/all/the/way/in/here/please/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/imdb/deep-package-path/all/the/way/in/here/please/internal/retrier.go +++ b/seed/go-sdk/imdb/deep-package-path/all/the/way/in/here/please/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/imdb/deep-package-path/all/the/way/in/here/please/internal/retrier_test.go b/seed/go-sdk/imdb/deep-package-path/all/the/way/in/here/please/internal/retrier_test.go index ec91b2cf678e..5380a9b676ce 100644 --- a/seed/go-sdk/imdb/deep-package-path/all/the/way/in/here/please/internal/retrier_test.go +++ b/seed/go-sdk/imdb/deep-package-path/all/the/way/in/here/please/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/imdb/include-platform-headers-app-info/internal/retrier.go b/seed/go-sdk/imdb/include-platform-headers-app-info/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/imdb/include-platform-headers-app-info/internal/retrier.go +++ b/seed/go-sdk/imdb/include-platform-headers-app-info/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/imdb/include-platform-headers-app-info/internal/retrier_test.go b/seed/go-sdk/imdb/include-platform-headers-app-info/internal/retrier_test.go index b9e74b043e11..09b373a2a3b5 100644 --- a/seed/go-sdk/imdb/include-platform-headers-app-info/internal/retrier_test.go +++ b/seed/go-sdk/imdb/include-platform-headers-app-info/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/imdb/include-platform-headers/internal/retrier.go b/seed/go-sdk/imdb/include-platform-headers/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/imdb/include-platform-headers/internal/retrier.go +++ b/seed/go-sdk/imdb/include-platform-headers/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/imdb/include-platform-headers/internal/retrier_test.go b/seed/go-sdk/imdb/include-platform-headers/internal/retrier_test.go index b9e74b043e11..09b373a2a3b5 100644 --- a/seed/go-sdk/imdb/include-platform-headers/internal/retrier_test.go +++ b/seed/go-sdk/imdb/include-platform-headers/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/imdb/no-custom-config/internal/retrier.go b/seed/go-sdk/imdb/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/imdb/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/imdb/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/imdb/no-custom-config/internal/retrier_test.go b/seed/go-sdk/imdb/no-custom-config/internal/retrier_test.go index b9e74b043e11..09b373a2a3b5 100644 --- a/seed/go-sdk/imdb/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/imdb/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/imdb/omit-fern-headers/internal/retrier.go b/seed/go-sdk/imdb/omit-fern-headers/internal/retrier.go index 6f1d2262a9b6..b0a271b495bd 100644 --- a/seed/go-sdk/imdb/omit-fern-headers/internal/retrier.go +++ b/seed/go-sdk/imdb/omit-fern-headers/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/imdb/omit-fern-headers/internal/retrier_test.go b/seed/go-sdk/imdb/omit-fern-headers/internal/retrier_test.go index b9e74b043e11..09b373a2a3b5 100644 --- a/seed/go-sdk/imdb/omit-fern-headers/internal/retrier_test.go +++ b/seed/go-sdk/imdb/omit-fern-headers/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/imdb/package-path/inhereplease/internal/retrier.go b/seed/go-sdk/imdb/package-path/inhereplease/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/imdb/package-path/inhereplease/internal/retrier.go +++ b/seed/go-sdk/imdb/package-path/inhereplease/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/imdb/package-path/inhereplease/internal/retrier_test.go b/seed/go-sdk/imdb/package-path/inhereplease/internal/retrier_test.go index 55e6f5312a79..651a5bab2c3a 100644 --- a/seed/go-sdk/imdb/package-path/inhereplease/internal/retrier_test.go +++ b/seed/go-sdk/imdb/package-path/inhereplease/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/imdb/with-wiremock-tests/internal/retrier.go b/seed/go-sdk/imdb/with-wiremock-tests/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/imdb/with-wiremock-tests/internal/retrier.go +++ b/seed/go-sdk/imdb/with-wiremock-tests/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/imdb/with-wiremock-tests/internal/retrier_test.go b/seed/go-sdk/imdb/with-wiremock-tests/internal/retrier_test.go index b9e74b043e11..09b373a2a3b5 100644 --- a/seed/go-sdk/imdb/with-wiremock-tests/internal/retrier_test.go +++ b/seed/go-sdk/imdb/with-wiremock-tests/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/inferred-auth-explicit/internal/retrier.go b/seed/go-sdk/inferred-auth-explicit/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/inferred-auth-explicit/internal/retrier.go +++ b/seed/go-sdk/inferred-auth-explicit/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/inferred-auth-explicit/internal/retrier_test.go b/seed/go-sdk/inferred-auth-explicit/internal/retrier_test.go index 7751098d8d79..c05548604485 100644 --- a/seed/go-sdk/inferred-auth-explicit/internal/retrier_test.go +++ b/seed/go-sdk/inferred-auth-explicit/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/inferred-auth-implicit-api-key/internal/retrier.go b/seed/go-sdk/inferred-auth-implicit-api-key/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/inferred-auth-implicit-api-key/internal/retrier.go +++ b/seed/go-sdk/inferred-auth-implicit-api-key/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/inferred-auth-implicit-api-key/internal/retrier_test.go b/seed/go-sdk/inferred-auth-implicit-api-key/internal/retrier_test.go index 1c4ff3c353f3..cce3d7739805 100644 --- a/seed/go-sdk/inferred-auth-implicit-api-key/internal/retrier_test.go +++ b/seed/go-sdk/inferred-auth-implicit-api-key/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/inferred-auth-implicit-no-expiry/internal/retrier.go b/seed/go-sdk/inferred-auth-implicit-no-expiry/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/inferred-auth-implicit-no-expiry/internal/retrier.go +++ b/seed/go-sdk/inferred-auth-implicit-no-expiry/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/inferred-auth-implicit-no-expiry/internal/retrier_test.go b/seed/go-sdk/inferred-auth-implicit-no-expiry/internal/retrier_test.go index 159cd3d74f14..d5b5a5215136 100644 --- a/seed/go-sdk/inferred-auth-implicit-no-expiry/internal/retrier_test.go +++ b/seed/go-sdk/inferred-auth-implicit-no-expiry/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/inferred-auth-implicit-reference/internal/retrier.go b/seed/go-sdk/inferred-auth-implicit-reference/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/inferred-auth-implicit-reference/internal/retrier.go +++ b/seed/go-sdk/inferred-auth-implicit-reference/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/inferred-auth-implicit-reference/internal/retrier_test.go b/seed/go-sdk/inferred-auth-implicit-reference/internal/retrier_test.go index 3356c9451077..3c9b3290adc5 100644 --- a/seed/go-sdk/inferred-auth-implicit-reference/internal/retrier_test.go +++ b/seed/go-sdk/inferred-auth-implicit-reference/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/inferred-auth-implicit/internal/retrier.go b/seed/go-sdk/inferred-auth-implicit/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/inferred-auth-implicit/internal/retrier.go +++ b/seed/go-sdk/inferred-auth-implicit/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/inferred-auth-implicit/internal/retrier_test.go b/seed/go-sdk/inferred-auth-implicit/internal/retrier_test.go index bdc8d1ccad79..51c057d8eb5c 100644 --- a/seed/go-sdk/inferred-auth-implicit/internal/retrier_test.go +++ b/seed/go-sdk/inferred-auth-implicit/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/inline-enum-type-name-override/internal/retrier.go b/seed/go-sdk/inline-enum-type-name-override/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/inline-enum-type-name-override/internal/retrier.go +++ b/seed/go-sdk/inline-enum-type-name-override/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/inline-enum-type-name-override/internal/retrier_test.go b/seed/go-sdk/inline-enum-type-name-override/internal/retrier_test.go index 6c6021a465cd..31f3ff8f4902 100644 --- a/seed/go-sdk/inline-enum-type-name-override/internal/retrier_test.go +++ b/seed/go-sdk/inline-enum-type-name-override/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/license/internal/retrier.go b/seed/go-sdk/license/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/license/internal/retrier.go +++ b/seed/go-sdk/license/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/license/internal/retrier_test.go b/seed/go-sdk/license/internal/retrier_test.go index 5fd097968920..cec2673f5e44 100644 --- a/seed/go-sdk/license/internal/retrier_test.go +++ b/seed/go-sdk/license/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/literal-user-agent/internal/retrier.go b/seed/go-sdk/literal-user-agent/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/literal-user-agent/internal/retrier.go +++ b/seed/go-sdk/literal-user-agent/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/literal-user-agent/internal/retrier_test.go b/seed/go-sdk/literal-user-agent/internal/retrier_test.go index 1c05bbee8c19..65ee0ce6106a 100644 --- a/seed/go-sdk/literal-user-agent/internal/retrier_test.go +++ b/seed/go-sdk/literal-user-agent/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/literal/internal/retrier.go b/seed/go-sdk/literal/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/literal/internal/retrier.go +++ b/seed/go-sdk/literal/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/literal/internal/retrier_test.go b/seed/go-sdk/literal/internal/retrier_test.go index 31c706f1a7ec..7ba02e8abf0e 100644 --- a/seed/go-sdk/literal/internal/retrier_test.go +++ b/seed/go-sdk/literal/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/literals-unions/no-custom-config/internal/retrier.go b/seed/go-sdk/literals-unions/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/literals-unions/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/literals-unions/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/literals-unions/no-custom-config/internal/retrier_test.go b/seed/go-sdk/literals-unions/no-custom-config/internal/retrier_test.go index 68d04122eedf..0742b8ffca51 100644 --- a/seed/go-sdk/literals-unions/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/literals-unions/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/mixed-case/default-values/internal/retrier.go b/seed/go-sdk/mixed-case/default-values/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/mixed-case/default-values/internal/retrier.go +++ b/seed/go-sdk/mixed-case/default-values/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/mixed-case/default-values/internal/retrier_test.go b/seed/go-sdk/mixed-case/default-values/internal/retrier_test.go index e00962fb120a..eb5f87b91036 100644 --- a/seed/go-sdk/mixed-case/default-values/internal/retrier_test.go +++ b/seed/go-sdk/mixed-case/default-values/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/mixed-case/no-custom-config/internal/retrier.go b/seed/go-sdk/mixed-case/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/mixed-case/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/mixed-case/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/mixed-case/no-custom-config/internal/retrier_test.go b/seed/go-sdk/mixed-case/no-custom-config/internal/retrier_test.go index e00962fb120a..eb5f87b91036 100644 --- a/seed/go-sdk/mixed-case/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/mixed-case/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/mixed-file-directory/internal/retrier.go b/seed/go-sdk/mixed-file-directory/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/mixed-file-directory/internal/retrier.go +++ b/seed/go-sdk/mixed-file-directory/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/mixed-file-directory/internal/retrier_test.go b/seed/go-sdk/mixed-file-directory/internal/retrier_test.go index 6e7c5eb8a976..26fc847081ef 100644 --- a/seed/go-sdk/mixed-file-directory/internal/retrier_test.go +++ b/seed/go-sdk/mixed-file-directory/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/multi-content-type-examples/internal/retrier.go b/seed/go-sdk/multi-content-type-examples/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/multi-content-type-examples/internal/retrier.go +++ b/seed/go-sdk/multi-content-type-examples/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/multi-content-type-examples/internal/retrier_test.go b/seed/go-sdk/multi-content-type-examples/internal/retrier_test.go index 338730599ab4..eca44126307c 100644 --- a/seed/go-sdk/multi-content-type-examples/internal/retrier_test.go +++ b/seed/go-sdk/multi-content-type-examples/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/multi-line-docs/internal/retrier.go b/seed/go-sdk/multi-line-docs/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/multi-line-docs/internal/retrier.go +++ b/seed/go-sdk/multi-line-docs/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/multi-line-docs/internal/retrier_test.go b/seed/go-sdk/multi-line-docs/internal/retrier_test.go index 3b412ec86996..9f5250b8a1e7 100644 --- a/seed/go-sdk/multi-line-docs/internal/retrier_test.go +++ b/seed/go-sdk/multi-line-docs/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/multi-url-environment-no-default/internal/retrier.go b/seed/go-sdk/multi-url-environment-no-default/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/multi-url-environment-no-default/internal/retrier.go +++ b/seed/go-sdk/multi-url-environment-no-default/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/multi-url-environment-no-default/internal/retrier_test.go b/seed/go-sdk/multi-url-environment-no-default/internal/retrier_test.go index 1be707b84019..95bfa2afeebf 100644 --- a/seed/go-sdk/multi-url-environment-no-default/internal/retrier_test.go +++ b/seed/go-sdk/multi-url-environment-no-default/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/multi-url-environment-reference/internal/retrier.go b/seed/go-sdk/multi-url-environment-reference/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/multi-url-environment-reference/internal/retrier.go +++ b/seed/go-sdk/multi-url-environment-reference/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/multi-url-environment-reference/internal/retrier_test.go b/seed/go-sdk/multi-url-environment-reference/internal/retrier_test.go index cc8a253773a5..b8cc3f71c9c7 100644 --- a/seed/go-sdk/multi-url-environment-reference/internal/retrier_test.go +++ b/seed/go-sdk/multi-url-environment-reference/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/multi-url-environment/internal/retrier.go b/seed/go-sdk/multi-url-environment/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/multi-url-environment/internal/retrier.go +++ b/seed/go-sdk/multi-url-environment/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/multi-url-environment/internal/retrier_test.go b/seed/go-sdk/multi-url-environment/internal/retrier_test.go index 792883342475..fb667f29220f 100644 --- a/seed/go-sdk/multi-url-environment/internal/retrier_test.go +++ b/seed/go-sdk/multi-url-environment/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/multiple-request-bodies/internal/retrier.go b/seed/go-sdk/multiple-request-bodies/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/multiple-request-bodies/internal/retrier.go +++ b/seed/go-sdk/multiple-request-bodies/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/multiple-request-bodies/internal/retrier_test.go b/seed/go-sdk/multiple-request-bodies/internal/retrier_test.go index acffeb95e694..51fd76e0c535 100644 --- a/seed/go-sdk/multiple-request-bodies/internal/retrier_test.go +++ b/seed/go-sdk/multiple-request-bodies/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/no-content-response/internal/retrier.go b/seed/go-sdk/no-content-response/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/no-content-response/internal/retrier.go +++ b/seed/go-sdk/no-content-response/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/no-content-response/internal/retrier_test.go b/seed/go-sdk/no-content-response/internal/retrier_test.go index cf02aaf8f5fb..4019ad862d54 100644 --- a/seed/go-sdk/no-content-response/internal/retrier_test.go +++ b/seed/go-sdk/no-content-response/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/no-environment/internal/retrier.go b/seed/go-sdk/no-environment/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/no-environment/internal/retrier.go +++ b/seed/go-sdk/no-environment/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/no-environment/internal/retrier_test.go b/seed/go-sdk/no-environment/internal/retrier_test.go index 810481f33dff..296e7e2f996d 100644 --- a/seed/go-sdk/no-environment/internal/retrier_test.go +++ b/seed/go-sdk/no-environment/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/no-retries/internal/retrier.go b/seed/go-sdk/no-retries/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/no-retries/internal/retrier.go +++ b/seed/go-sdk/no-retries/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/no-retries/internal/retrier_test.go b/seed/go-sdk/no-retries/internal/retrier_test.go index 739426dc63f4..0c22f0d52f7f 100644 --- a/seed/go-sdk/no-retries/internal/retrier_test.go +++ b/seed/go-sdk/no-retries/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/null-type/internal/retrier.go b/seed/go-sdk/null-type/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/null-type/internal/retrier.go +++ b/seed/go-sdk/null-type/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/null-type/internal/retrier_test.go b/seed/go-sdk/null-type/internal/retrier_test.go index 85e49cd325b4..6583551c9482 100644 --- a/seed/go-sdk/null-type/internal/retrier_test.go +++ b/seed/go-sdk/null-type/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/nullable-allof-extends/internal/retrier.go b/seed/go-sdk/nullable-allof-extends/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/nullable-allof-extends/internal/retrier.go +++ b/seed/go-sdk/nullable-allof-extends/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/nullable-allof-extends/internal/retrier_test.go b/seed/go-sdk/nullable-allof-extends/internal/retrier_test.go index 9d5b2f58748a..e42ba15ff99f 100644 --- a/seed/go-sdk/nullable-allof-extends/internal/retrier_test.go +++ b/seed/go-sdk/nullable-allof-extends/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/nullable-optional/internal/retrier.go b/seed/go-sdk/nullable-optional/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/nullable-optional/internal/retrier.go +++ b/seed/go-sdk/nullable-optional/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/nullable-optional/internal/retrier_test.go b/seed/go-sdk/nullable-optional/internal/retrier_test.go index 75462543a0c0..3501ef76e478 100644 --- a/seed/go-sdk/nullable-optional/internal/retrier_test.go +++ b/seed/go-sdk/nullable-optional/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/nullable-request-body/dynamic-snippets-disabled/internal/retrier.go b/seed/go-sdk/nullable-request-body/dynamic-snippets-disabled/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/nullable-request-body/dynamic-snippets-disabled/internal/retrier.go +++ b/seed/go-sdk/nullable-request-body/dynamic-snippets-disabled/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/nullable-request-body/dynamic-snippets-disabled/internal/retrier_test.go b/seed/go-sdk/nullable-request-body/dynamic-snippets-disabled/internal/retrier_test.go index 0502d78fd6d8..d794c804f28d 100644 --- a/seed/go-sdk/nullable-request-body/dynamic-snippets-disabled/internal/retrier_test.go +++ b/seed/go-sdk/nullable-request-body/dynamic-snippets-disabled/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/nullable/internal/retrier.go b/seed/go-sdk/nullable/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/nullable/internal/retrier.go +++ b/seed/go-sdk/nullable/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/nullable/internal/retrier_test.go b/seed/go-sdk/nullable/internal/retrier_test.go index 9c95b8d988f6..840f87addd96 100644 --- a/seed/go-sdk/nullable/internal/retrier_test.go +++ b/seed/go-sdk/nullable/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/oauth-client-credentials-custom/internal/retrier.go b/seed/go-sdk/oauth-client-credentials-custom/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/oauth-client-credentials-custom/internal/retrier.go +++ b/seed/go-sdk/oauth-client-credentials-custom/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/oauth-client-credentials-custom/internal/retrier_test.go b/seed/go-sdk/oauth-client-credentials-custom/internal/retrier_test.go index 887d7df97950..ec041ac3d914 100644 --- a/seed/go-sdk/oauth-client-credentials-custom/internal/retrier_test.go +++ b/seed/go-sdk/oauth-client-credentials-custom/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/oauth-client-credentials-default/internal/retrier.go b/seed/go-sdk/oauth-client-credentials-default/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/oauth-client-credentials-default/internal/retrier.go +++ b/seed/go-sdk/oauth-client-credentials-default/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/oauth-client-credentials-default/internal/retrier_test.go b/seed/go-sdk/oauth-client-credentials-default/internal/retrier_test.go index 1afb1aa56ecf..a820354290bd 100644 --- a/seed/go-sdk/oauth-client-credentials-default/internal/retrier_test.go +++ b/seed/go-sdk/oauth-client-credentials-default/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/oauth-client-credentials-environment-variables/internal/retrier.go b/seed/go-sdk/oauth-client-credentials-environment-variables/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/oauth-client-credentials-environment-variables/internal/retrier.go +++ b/seed/go-sdk/oauth-client-credentials-environment-variables/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/oauth-client-credentials-environment-variables/internal/retrier_test.go b/seed/go-sdk/oauth-client-credentials-environment-variables/internal/retrier_test.go index 808661967104..f470e149a086 100644 --- a/seed/go-sdk/oauth-client-credentials-environment-variables/internal/retrier_test.go +++ b/seed/go-sdk/oauth-client-credentials-environment-variables/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/oauth-client-credentials-mandatory-auth/no-custom-config/internal/retrier.go b/seed/go-sdk/oauth-client-credentials-mandatory-auth/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/oauth-client-credentials-mandatory-auth/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/oauth-client-credentials-mandatory-auth/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/oauth-client-credentials-mandatory-auth/no-custom-config/internal/retrier_test.go b/seed/go-sdk/oauth-client-credentials-mandatory-auth/no-custom-config/internal/retrier_test.go index 2d485ebd7752..dcd552be4bff 100644 --- a/seed/go-sdk/oauth-client-credentials-mandatory-auth/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/oauth-client-credentials-mandatory-auth/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/oauth-client-credentials-nested-root/internal/retrier.go b/seed/go-sdk/oauth-client-credentials-nested-root/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/oauth-client-credentials-nested-root/internal/retrier.go +++ b/seed/go-sdk/oauth-client-credentials-nested-root/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/oauth-client-credentials-nested-root/internal/retrier_test.go b/seed/go-sdk/oauth-client-credentials-nested-root/internal/retrier_test.go index 3f360834bc36..5fb54d107521 100644 --- a/seed/go-sdk/oauth-client-credentials-nested-root/internal/retrier_test.go +++ b/seed/go-sdk/oauth-client-credentials-nested-root/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/oauth-client-credentials-openapi/internal/retrier.go b/seed/go-sdk/oauth-client-credentials-openapi/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/oauth-client-credentials-openapi/internal/retrier.go +++ b/seed/go-sdk/oauth-client-credentials-openapi/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/oauth-client-credentials-openapi/internal/retrier_test.go b/seed/go-sdk/oauth-client-credentials-openapi/internal/retrier_test.go index 992c337cdeeb..913b9ac619de 100644 --- a/seed/go-sdk/oauth-client-credentials-openapi/internal/retrier_test.go +++ b/seed/go-sdk/oauth-client-credentials-openapi/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/oauth-client-credentials-reference/internal/retrier.go b/seed/go-sdk/oauth-client-credentials-reference/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/oauth-client-credentials-reference/internal/retrier.go +++ b/seed/go-sdk/oauth-client-credentials-reference/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/oauth-client-credentials-reference/internal/retrier_test.go b/seed/go-sdk/oauth-client-credentials-reference/internal/retrier_test.go index b2e53e4cd80c..837b1d2fe46a 100644 --- a/seed/go-sdk/oauth-client-credentials-reference/internal/retrier_test.go +++ b/seed/go-sdk/oauth-client-credentials-reference/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/oauth-client-credentials-with-variables/internal/retrier.go b/seed/go-sdk/oauth-client-credentials-with-variables/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/oauth-client-credentials-with-variables/internal/retrier.go +++ b/seed/go-sdk/oauth-client-credentials-with-variables/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/oauth-client-credentials-with-variables/internal/retrier_test.go b/seed/go-sdk/oauth-client-credentials-with-variables/internal/retrier_test.go index 5a6950a9b615..909bfc546872 100644 --- a/seed/go-sdk/oauth-client-credentials-with-variables/internal/retrier_test.go +++ b/seed/go-sdk/oauth-client-credentials-with-variables/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/oauth-client-credentials/internal/retrier.go b/seed/go-sdk/oauth-client-credentials/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/oauth-client-credentials/internal/retrier.go +++ b/seed/go-sdk/oauth-client-credentials/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/oauth-client-credentials/internal/retrier_test.go b/seed/go-sdk/oauth-client-credentials/internal/retrier_test.go index 6119b0ef71d8..246f84a27021 100644 --- a/seed/go-sdk/oauth-client-credentials/internal/retrier_test.go +++ b/seed/go-sdk/oauth-client-credentials/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/oauth-pkce/internal/retrier.go b/seed/go-sdk/oauth-pkce/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/oauth-pkce/internal/retrier.go +++ b/seed/go-sdk/oauth-pkce/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/oauth-pkce/internal/retrier_test.go b/seed/go-sdk/oauth-pkce/internal/retrier_test.go index e814414cbe51..7abc8b84819a 100644 --- a/seed/go-sdk/oauth-pkce/internal/retrier_test.go +++ b/seed/go-sdk/oauth-pkce/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/object/internal/retrier.go b/seed/go-sdk/object/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/object/internal/retrier.go +++ b/seed/go-sdk/object/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/object/internal/retrier_test.go b/seed/go-sdk/object/internal/retrier_test.go index 9b94fea25f54..8fdc3782fa5b 100644 --- a/seed/go-sdk/object/internal/retrier_test.go +++ b/seed/go-sdk/object/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/objects-with-imports/internal/retrier.go b/seed/go-sdk/objects-with-imports/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/objects-with-imports/internal/retrier.go +++ b/seed/go-sdk/objects-with-imports/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/objects-with-imports/internal/retrier_test.go b/seed/go-sdk/objects-with-imports/internal/retrier_test.go index 2742dd8bf14c..f008257324c2 100644 --- a/seed/go-sdk/objects-with-imports/internal/retrier_test.go +++ b/seed/go-sdk/objects-with-imports/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/openapi-path-param-body-collision/internal/retrier.go b/seed/go-sdk/openapi-path-param-body-collision/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/openapi-path-param-body-collision/internal/retrier.go +++ b/seed/go-sdk/openapi-path-param-body-collision/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/openapi-path-param-body-collision/internal/retrier_test.go b/seed/go-sdk/openapi-path-param-body-collision/internal/retrier_test.go index dd8b4183e9c9..1bec8cf68c1e 100644 --- a/seed/go-sdk/openapi-path-param-body-collision/internal/retrier_test.go +++ b/seed/go-sdk/openapi-path-param-body-collision/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/openapi-request-body-ref/no-custom-config/internal/retrier.go b/seed/go-sdk/openapi-request-body-ref/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/openapi-request-body-ref/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/openapi-request-body-ref/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/openapi-request-body-ref/no-custom-config/internal/retrier_test.go b/seed/go-sdk/openapi-request-body-ref/no-custom-config/internal/retrier_test.go index d380d588d628..dc53fdf499cb 100644 --- a/seed/go-sdk/openapi-request-body-ref/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/openapi-request-body-ref/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/openapi-subtitle/internal/retrier.go b/seed/go-sdk/openapi-subtitle/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/openapi-subtitle/internal/retrier.go +++ b/seed/go-sdk/openapi-subtitle/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/openapi-subtitle/internal/retrier_test.go b/seed/go-sdk/openapi-subtitle/internal/retrier_test.go index 92b1e0acab59..abd6767b956a 100644 --- a/seed/go-sdk/openapi-subtitle/internal/retrier_test.go +++ b/seed/go-sdk/openapi-subtitle/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/optional/internal/retrier.go b/seed/go-sdk/optional/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/optional/internal/retrier.go +++ b/seed/go-sdk/optional/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/optional/internal/retrier_test.go b/seed/go-sdk/optional/internal/retrier_test.go index 882f0161cbc7..20b56647585b 100644 --- a/seed/go-sdk/optional/internal/retrier_test.go +++ b/seed/go-sdk/optional/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/package-yml/no-custom-config/internal/retrier.go b/seed/go-sdk/package-yml/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/package-yml/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/package-yml/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/package-yml/no-custom-config/internal/retrier_test.go b/seed/go-sdk/package-yml/no-custom-config/internal/retrier_test.go index ca22e2b0295e..109e4cdf69a7 100644 --- a/seed/go-sdk/package-yml/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/package-yml/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/pagination-custom/internal/retrier.go b/seed/go-sdk/pagination-custom/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/pagination-custom/internal/retrier.go +++ b/seed/go-sdk/pagination-custom/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/pagination-custom/internal/retrier_test.go b/seed/go-sdk/pagination-custom/internal/retrier_test.go index b45747402e02..d215abee9253 100644 --- a/seed/go-sdk/pagination-custom/internal/retrier_test.go +++ b/seed/go-sdk/pagination-custom/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/pagination-uri-path/internal/retrier.go b/seed/go-sdk/pagination-uri-path/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/pagination-uri-path/internal/retrier.go +++ b/seed/go-sdk/pagination-uri-path/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/pagination-uri-path/internal/retrier_test.go b/seed/go-sdk/pagination-uri-path/internal/retrier_test.go index 07637d62a3c5..850709e227ce 100644 --- a/seed/go-sdk/pagination-uri-path/internal/retrier_test.go +++ b/seed/go-sdk/pagination-uri-path/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/pagination/internal/retrier.go b/seed/go-sdk/pagination/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/pagination/internal/retrier.go +++ b/seed/go-sdk/pagination/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/pagination/internal/retrier_test.go b/seed/go-sdk/pagination/internal/retrier_test.go index 014e9eb13f58..4c6337fa7abf 100644 --- a/seed/go-sdk/pagination/internal/retrier_test.go +++ b/seed/go-sdk/pagination/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/path-parameters/no-custom-config/internal/retrier.go b/seed/go-sdk/path-parameters/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/path-parameters/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/path-parameters/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/path-parameters/no-custom-config/internal/retrier_test.go b/seed/go-sdk/path-parameters/no-custom-config/internal/retrier_test.go index 57cd6e8b0b80..ae023a4e7268 100644 --- a/seed/go-sdk/path-parameters/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/path-parameters/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/path-parameters/package-name/internal/retrier.go b/seed/go-sdk/path-parameters/package-name/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/path-parameters/package-name/internal/retrier.go +++ b/seed/go-sdk/path-parameters/package-name/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/path-parameters/package-name/internal/retrier_test.go b/seed/go-sdk/path-parameters/package-name/internal/retrier_test.go index 5654b65f21c5..6c5abb4d9ffb 100644 --- a/seed/go-sdk/path-parameters/package-name/internal/retrier_test.go +++ b/seed/go-sdk/path-parameters/package-name/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/path-parameters/v0/internal/retrier.go b/seed/go-sdk/path-parameters/v0/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/path-parameters/v0/internal/retrier.go +++ b/seed/go-sdk/path-parameters/v0/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/path-parameters/v0/internal/retrier_test.go b/seed/go-sdk/path-parameters/v0/internal/retrier_test.go index 57cd6e8b0b80..ae023a4e7268 100644 --- a/seed/go-sdk/path-parameters/v0/internal/retrier_test.go +++ b/seed/go-sdk/path-parameters/v0/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/plain-text/internal/retrier.go b/seed/go-sdk/plain-text/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/plain-text/internal/retrier.go +++ b/seed/go-sdk/plain-text/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/plain-text/internal/retrier_test.go b/seed/go-sdk/plain-text/internal/retrier_test.go index bc0ba088cac8..e4b38d56808f 100644 --- a/seed/go-sdk/plain-text/internal/retrier_test.go +++ b/seed/go-sdk/plain-text/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/property-access/internal/retrier.go b/seed/go-sdk/property-access/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/property-access/internal/retrier.go +++ b/seed/go-sdk/property-access/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/property-access/internal/retrier_test.go b/seed/go-sdk/property-access/internal/retrier_test.go index 8fcbb9c57b1a..128622f328b4 100644 --- a/seed/go-sdk/property-access/internal/retrier_test.go +++ b/seed/go-sdk/property-access/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/public-object/internal/retrier.go b/seed/go-sdk/public-object/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/public-object/internal/retrier.go +++ b/seed/go-sdk/public-object/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/public-object/internal/retrier_test.go b/seed/go-sdk/public-object/internal/retrier_test.go index 6789c2d66c17..d673214df936 100644 --- a/seed/go-sdk/public-object/internal/retrier_test.go +++ b/seed/go-sdk/public-object/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/query-param-name-conflict/internal/retrier.go b/seed/go-sdk/query-param-name-conflict/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/query-param-name-conflict/internal/retrier.go +++ b/seed/go-sdk/query-param-name-conflict/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/query-param-name-conflict/internal/retrier_test.go b/seed/go-sdk/query-param-name-conflict/internal/retrier_test.go index 1e30db1499c3..792bc95619b8 100644 --- a/seed/go-sdk/query-param-name-conflict/internal/retrier_test.go +++ b/seed/go-sdk/query-param-name-conflict/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/query-parameters-openapi-as-objects/internal/retrier.go b/seed/go-sdk/query-parameters-openapi-as-objects/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/query-parameters-openapi-as-objects/internal/retrier.go +++ b/seed/go-sdk/query-parameters-openapi-as-objects/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/query-parameters-openapi-as-objects/internal/retrier_test.go b/seed/go-sdk/query-parameters-openapi-as-objects/internal/retrier_test.go index e533e758bfec..22394b3d0a89 100644 --- a/seed/go-sdk/query-parameters-openapi-as-objects/internal/retrier_test.go +++ b/seed/go-sdk/query-parameters-openapi-as-objects/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/query-parameters-openapi/internal/retrier.go b/seed/go-sdk/query-parameters-openapi/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/query-parameters-openapi/internal/retrier.go +++ b/seed/go-sdk/query-parameters-openapi/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/query-parameters-openapi/internal/retrier_test.go b/seed/go-sdk/query-parameters-openapi/internal/retrier_test.go index b982caf5d1b9..1f418f509c5f 100644 --- a/seed/go-sdk/query-parameters-openapi/internal/retrier_test.go +++ b/seed/go-sdk/query-parameters-openapi/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/query-parameters/internal/retrier.go b/seed/go-sdk/query-parameters/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/query-parameters/internal/retrier.go +++ b/seed/go-sdk/query-parameters/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/query-parameters/internal/retrier_test.go b/seed/go-sdk/query-parameters/internal/retrier_test.go index 3b14eed6d9bd..bb156a0487aa 100644 --- a/seed/go-sdk/query-parameters/internal/retrier_test.go +++ b/seed/go-sdk/query-parameters/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/required-nullable/internal/retrier.go b/seed/go-sdk/required-nullable/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/required-nullable/internal/retrier.go +++ b/seed/go-sdk/required-nullable/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/required-nullable/internal/retrier_test.go b/seed/go-sdk/required-nullable/internal/retrier_test.go index 28c34a3aad52..6f6abb92a9e2 100644 --- a/seed/go-sdk/required-nullable/internal/retrier_test.go +++ b/seed/go-sdk/required-nullable/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/reserved-keywords/internal/retrier.go b/seed/go-sdk/reserved-keywords/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/reserved-keywords/internal/retrier.go +++ b/seed/go-sdk/reserved-keywords/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/reserved-keywords/internal/retrier_test.go b/seed/go-sdk/reserved-keywords/internal/retrier_test.go index 7586160379fd..9708121b4655 100644 --- a/seed/go-sdk/reserved-keywords/internal/retrier_test.go +++ b/seed/go-sdk/reserved-keywords/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/respect-optional-request-body/internal/retrier.go b/seed/go-sdk/respect-optional-request-body/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/respect-optional-request-body/internal/retrier.go +++ b/seed/go-sdk/respect-optional-request-body/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/respect-optional-request-body/internal/retrier_test.go b/seed/go-sdk/respect-optional-request-body/internal/retrier_test.go index 426b3b89340c..fedd10506f66 100644 --- a/seed/go-sdk/respect-optional-request-body/internal/retrier_test.go +++ b/seed/go-sdk/respect-optional-request-body/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/response-property/internal/retrier.go b/seed/go-sdk/response-property/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/response-property/internal/retrier.go +++ b/seed/go-sdk/response-property/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/response-property/internal/retrier_test.go b/seed/go-sdk/response-property/internal/retrier_test.go index fb7fc1db3112..d513a3bf814c 100644 --- a/seed/go-sdk/response-property/internal/retrier_test.go +++ b/seed/go-sdk/response-property/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/schemaless-request-body-examples/internal/retrier.go b/seed/go-sdk/schemaless-request-body-examples/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/schemaless-request-body-examples/internal/retrier.go +++ b/seed/go-sdk/schemaless-request-body-examples/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/schemaless-request-body-examples/internal/retrier_test.go b/seed/go-sdk/schemaless-request-body-examples/internal/retrier_test.go index db6915c0e695..024c96f09582 100644 --- a/seed/go-sdk/schemaless-request-body-examples/internal/retrier_test.go +++ b/seed/go-sdk/schemaless-request-body-examples/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/server-sent-event-examples/with-wire-tests/internal/retrier.go b/seed/go-sdk/server-sent-event-examples/with-wire-tests/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/server-sent-event-examples/with-wire-tests/internal/retrier.go +++ b/seed/go-sdk/server-sent-event-examples/with-wire-tests/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/server-sent-event-examples/with-wire-tests/internal/retrier_test.go b/seed/go-sdk/server-sent-event-examples/with-wire-tests/internal/retrier_test.go index 22fa39eb0d42..f1444d3f7107 100644 --- a/seed/go-sdk/server-sent-event-examples/with-wire-tests/internal/retrier_test.go +++ b/seed/go-sdk/server-sent-event-examples/with-wire-tests/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/server-sent-events-openapi/with-wire-tests/internal/retrier.go b/seed/go-sdk/server-sent-events-openapi/with-wire-tests/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/server-sent-events-openapi/with-wire-tests/internal/retrier.go +++ b/seed/go-sdk/server-sent-events-openapi/with-wire-tests/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/server-sent-events-openapi/with-wire-tests/internal/retrier_test.go b/seed/go-sdk/server-sent-events-openapi/with-wire-tests/internal/retrier_test.go index ef42d905509f..bb13be1bf191 100644 --- a/seed/go-sdk/server-sent-events-openapi/with-wire-tests/internal/retrier_test.go +++ b/seed/go-sdk/server-sent-events-openapi/with-wire-tests/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/server-sent-events-resumable/internal/retrier.go b/seed/go-sdk/server-sent-events-resumable/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/server-sent-events-resumable/internal/retrier.go +++ b/seed/go-sdk/server-sent-events-resumable/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/server-sent-events-resumable/internal/retrier_test.go b/seed/go-sdk/server-sent-events-resumable/internal/retrier_test.go index 4650aaafd3c6..282c999c5634 100644 --- a/seed/go-sdk/server-sent-events-resumable/internal/retrier_test.go +++ b/seed/go-sdk/server-sent-events-resumable/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/server-sent-events/with-wire-tests/internal/retrier.go b/seed/go-sdk/server-sent-events/with-wire-tests/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/server-sent-events/with-wire-tests/internal/retrier.go +++ b/seed/go-sdk/server-sent-events/with-wire-tests/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/server-sent-events/with-wire-tests/internal/retrier_test.go b/seed/go-sdk/server-sent-events/with-wire-tests/internal/retrier_test.go index 2408515e17d5..3b1808555960 100644 --- a/seed/go-sdk/server-sent-events/with-wire-tests/internal/retrier_test.go +++ b/seed/go-sdk/server-sent-events/with-wire-tests/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/server-url-templating-single-url/disable-server-url-variables/internal/retrier.go b/seed/go-sdk/server-url-templating-single-url/disable-server-url-variables/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/server-url-templating-single-url/disable-server-url-variables/internal/retrier.go +++ b/seed/go-sdk/server-url-templating-single-url/disable-server-url-variables/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/server-url-templating-single-url/disable-server-url-variables/internal/retrier_test.go b/seed/go-sdk/server-url-templating-single-url/disable-server-url-variables/internal/retrier_test.go index c007f11d0899..177957f7eb3b 100644 --- a/seed/go-sdk/server-url-templating-single-url/disable-server-url-variables/internal/retrier_test.go +++ b/seed/go-sdk/server-url-templating-single-url/disable-server-url-variables/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/server-url-templating-single-url/no-custom-config/internal/retrier.go b/seed/go-sdk/server-url-templating-single-url/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/server-url-templating-single-url/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/server-url-templating-single-url/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/server-url-templating-single-url/no-custom-config/internal/retrier_test.go b/seed/go-sdk/server-url-templating-single-url/no-custom-config/internal/retrier_test.go index c007f11d0899..177957f7eb3b 100644 --- a/seed/go-sdk/server-url-templating-single-url/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/server-url-templating-single-url/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/server-url-templating/disable-server-url-variables/internal/retrier.go b/seed/go-sdk/server-url-templating/disable-server-url-variables/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/server-url-templating/disable-server-url-variables/internal/retrier.go +++ b/seed/go-sdk/server-url-templating/disable-server-url-variables/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/server-url-templating/disable-server-url-variables/internal/retrier_test.go b/seed/go-sdk/server-url-templating/disable-server-url-variables/internal/retrier_test.go index 8351dd6b01a8..80870047533f 100644 --- a/seed/go-sdk/server-url-templating/disable-server-url-variables/internal/retrier_test.go +++ b/seed/go-sdk/server-url-templating/disable-server-url-variables/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/server-url-templating/no-custom-config/internal/retrier.go b/seed/go-sdk/server-url-templating/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/server-url-templating/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/server-url-templating/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/server-url-templating/no-custom-config/internal/retrier_test.go b/seed/go-sdk/server-url-templating/no-custom-config/internal/retrier_test.go index 8351dd6b01a8..80870047533f 100644 --- a/seed/go-sdk/server-url-templating/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/server-url-templating/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/simple-api/internal/retrier.go b/seed/go-sdk/simple-api/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/simple-api/internal/retrier.go +++ b/seed/go-sdk/simple-api/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/simple-api/internal/retrier_test.go b/seed/go-sdk/simple-api/internal/retrier_test.go index f3fd0be5cc87..1e1fc3a3d412 100644 --- a/seed/go-sdk/simple-api/internal/retrier_test.go +++ b/seed/go-sdk/simple-api/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/simple-fhir/internal/retrier.go b/seed/go-sdk/simple-fhir/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/simple-fhir/internal/retrier.go +++ b/seed/go-sdk/simple-fhir/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/simple-fhir/internal/retrier_test.go b/seed/go-sdk/simple-fhir/internal/retrier_test.go index 5d6d614c8cc6..d31b85bb9fb6 100644 --- a/seed/go-sdk/simple-fhir/internal/retrier_test.go +++ b/seed/go-sdk/simple-fhir/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/single-url-environment-default/internal/retrier.go b/seed/go-sdk/single-url-environment-default/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/single-url-environment-default/internal/retrier.go +++ b/seed/go-sdk/single-url-environment-default/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/single-url-environment-default/internal/retrier_test.go b/seed/go-sdk/single-url-environment-default/internal/retrier_test.go index 77eb7cab6935..b1919b6bb034 100644 --- a/seed/go-sdk/single-url-environment-default/internal/retrier_test.go +++ b/seed/go-sdk/single-url-environment-default/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/single-url-environment-no-default/internal/retrier.go b/seed/go-sdk/single-url-environment-no-default/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/single-url-environment-no-default/internal/retrier.go +++ b/seed/go-sdk/single-url-environment-no-default/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/single-url-environment-no-default/internal/retrier_test.go b/seed/go-sdk/single-url-environment-no-default/internal/retrier_test.go index 790671d6f6a4..83edcf244c13 100644 --- a/seed/go-sdk/single-url-environment-no-default/internal/retrier_test.go +++ b/seed/go-sdk/single-url-environment-no-default/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/streaming-parameter/internal/retrier.go b/seed/go-sdk/streaming-parameter/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/streaming-parameter/internal/retrier.go +++ b/seed/go-sdk/streaming-parameter/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/streaming-parameter/internal/retrier_test.go b/seed/go-sdk/streaming-parameter/internal/retrier_test.go index a9156925acf9..4f4c476ade9d 100644 --- a/seed/go-sdk/streaming-parameter/internal/retrier_test.go +++ b/seed/go-sdk/streaming-parameter/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/streaming/internal/retrier.go b/seed/go-sdk/streaming/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/streaming/internal/retrier.go +++ b/seed/go-sdk/streaming/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/streaming/internal/retrier_test.go b/seed/go-sdk/streaming/internal/retrier_test.go index d84b486fba25..084c15683340 100644 --- a/seed/go-sdk/streaming/internal/retrier_test.go +++ b/seed/go-sdk/streaming/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/trace/internal/retrier.go b/seed/go-sdk/trace/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/trace/internal/retrier.go +++ b/seed/go-sdk/trace/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/trace/internal/retrier_test.go b/seed/go-sdk/trace/internal/retrier_test.go index e02e7f94c630..9da7790f5209 100644 --- a/seed/go-sdk/trace/internal/retrier_test.go +++ b/seed/go-sdk/trace/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/undiscriminated-union-with-response-property/internal/retrier.go b/seed/go-sdk/undiscriminated-union-with-response-property/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/undiscriminated-union-with-response-property/internal/retrier.go +++ b/seed/go-sdk/undiscriminated-union-with-response-property/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/undiscriminated-union-with-response-property/internal/retrier_test.go b/seed/go-sdk/undiscriminated-union-with-response-property/internal/retrier_test.go index f12d0a9d13ec..d36a8834fa24 100644 --- a/seed/go-sdk/undiscriminated-union-with-response-property/internal/retrier_test.go +++ b/seed/go-sdk/undiscriminated-union-with-response-property/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/undiscriminated-unions/no-custom-config/internal/retrier.go b/seed/go-sdk/undiscriminated-unions/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/undiscriminated-unions/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/undiscriminated-unions/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/undiscriminated-unions/no-custom-config/internal/retrier_test.go b/seed/go-sdk/undiscriminated-unions/no-custom-config/internal/retrier_test.go index d81f23e2f1d2..4b91a7562bf4 100644 --- a/seed/go-sdk/undiscriminated-unions/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/undiscriminated-unions/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/undiscriminated-unions/v0/internal/retrier.go b/seed/go-sdk/undiscriminated-unions/v0/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/undiscriminated-unions/v0/internal/retrier.go +++ b/seed/go-sdk/undiscriminated-unions/v0/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/undiscriminated-unions/v0/internal/retrier_test.go b/seed/go-sdk/undiscriminated-unions/v0/internal/retrier_test.go index 25b03c193694..a896d388fd6a 100644 --- a/seed/go-sdk/undiscriminated-unions/v0/internal/retrier_test.go +++ b/seed/go-sdk/undiscriminated-unions/v0/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/union-query-parameters/internal/retrier.go b/seed/go-sdk/union-query-parameters/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/union-query-parameters/internal/retrier.go +++ b/seed/go-sdk/union-query-parameters/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/union-query-parameters/internal/retrier_test.go b/seed/go-sdk/union-query-parameters/internal/retrier_test.go index 3a3e3716aaed..df2b4355957e 100644 --- a/seed/go-sdk/union-query-parameters/internal/retrier_test.go +++ b/seed/go-sdk/union-query-parameters/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/unions-with-local-date/internal/retrier.go b/seed/go-sdk/unions-with-local-date/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/unions-with-local-date/internal/retrier.go +++ b/seed/go-sdk/unions-with-local-date/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/unions-with-local-date/internal/retrier_test.go b/seed/go-sdk/unions-with-local-date/internal/retrier_test.go index 4d022e82ec9e..f8c921524f24 100644 --- a/seed/go-sdk/unions-with-local-date/internal/retrier_test.go +++ b/seed/go-sdk/unions-with-local-date/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/unions/no-custom-config/internal/retrier.go b/seed/go-sdk/unions/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/unions/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/unions/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/unions/no-custom-config/internal/retrier_test.go b/seed/go-sdk/unions/no-custom-config/internal/retrier_test.go index 13af45d76301..61185830d1ed 100644 --- a/seed/go-sdk/unions/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/unions/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/unions/package-name/internal/retrier.go b/seed/go-sdk/unions/package-name/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/unions/package-name/internal/retrier.go +++ b/seed/go-sdk/unions/package-name/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/unions/package-name/internal/retrier_test.go b/seed/go-sdk/unions/package-name/internal/retrier_test.go index 0dace410ed60..36b0fea84e86 100644 --- a/seed/go-sdk/unions/package-name/internal/retrier_test.go +++ b/seed/go-sdk/unions/package-name/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/unions/v0/internal/retrier.go b/seed/go-sdk/unions/v0/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/unions/v0/internal/retrier.go +++ b/seed/go-sdk/unions/v0/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/unions/v0/internal/retrier_test.go b/seed/go-sdk/unions/v0/internal/retrier_test.go index 13af45d76301..61185830d1ed 100644 --- a/seed/go-sdk/unions/v0/internal/retrier_test.go +++ b/seed/go-sdk/unions/v0/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/unknown/internal/retrier.go b/seed/go-sdk/unknown/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/unknown/internal/retrier.go +++ b/seed/go-sdk/unknown/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/unknown/internal/retrier_test.go b/seed/go-sdk/unknown/internal/retrier_test.go index dedb8671e980..1d8b5ad4c7e2 100644 --- a/seed/go-sdk/unknown/internal/retrier_test.go +++ b/seed/go-sdk/unknown/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/url-form-encoded/internal/retrier.go b/seed/go-sdk/url-form-encoded/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/url-form-encoded/internal/retrier.go +++ b/seed/go-sdk/url-form-encoded/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/url-form-encoded/internal/retrier_test.go b/seed/go-sdk/url-form-encoded/internal/retrier_test.go index a6c58a1995b7..a6ff57ff4191 100644 --- a/seed/go-sdk/url-form-encoded/internal/retrier_test.go +++ b/seed/go-sdk/url-form-encoded/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/validation/internal/retrier.go b/seed/go-sdk/validation/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/validation/internal/retrier.go +++ b/seed/go-sdk/validation/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/validation/internal/retrier_test.go b/seed/go-sdk/validation/internal/retrier_test.go index 7bee6ed24860..5bbbbd744d2e 100644 --- a/seed/go-sdk/validation/internal/retrier_test.go +++ b/seed/go-sdk/validation/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/variables/internal/retrier.go b/seed/go-sdk/variables/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/variables/internal/retrier.go +++ b/seed/go-sdk/variables/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/variables/internal/retrier_test.go b/seed/go-sdk/variables/internal/retrier_test.go index f0c488f4bee0..8df1e6681639 100644 --- a/seed/go-sdk/variables/internal/retrier_test.go +++ b/seed/go-sdk/variables/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/version-no-default/internal/retrier.go b/seed/go-sdk/version-no-default/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/version-no-default/internal/retrier.go +++ b/seed/go-sdk/version-no-default/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/version-no-default/internal/retrier_test.go b/seed/go-sdk/version-no-default/internal/retrier_test.go index fca6e0888235..9c31a67f3b69 100644 --- a/seed/go-sdk/version-no-default/internal/retrier_test.go +++ b/seed/go-sdk/version-no-default/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/version/internal/retrier.go b/seed/go-sdk/version/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/version/internal/retrier.go +++ b/seed/go-sdk/version/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/version/internal/retrier_test.go b/seed/go-sdk/version/internal/retrier_test.go index 5a204c675a53..e65564f06064 100644 --- a/seed/go-sdk/version/internal/retrier_test.go +++ b/seed/go-sdk/version/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/webhook-audience/internal/retrier.go b/seed/go-sdk/webhook-audience/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/webhook-audience/internal/retrier.go +++ b/seed/go-sdk/webhook-audience/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/webhook-audience/internal/retrier_test.go b/seed/go-sdk/webhook-audience/internal/retrier_test.go index 64f8e20e742f..4f97b4f3d1a7 100644 --- a/seed/go-sdk/webhook-audience/internal/retrier_test.go +++ b/seed/go-sdk/webhook-audience/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/webhooks/internal/retrier.go b/seed/go-sdk/webhooks/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/webhooks/internal/retrier.go +++ b/seed/go-sdk/webhooks/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/webhooks/internal/retrier_test.go b/seed/go-sdk/webhooks/internal/retrier_test.go index 990af462517a..863c42350625 100644 --- a/seed/go-sdk/webhooks/internal/retrier_test.go +++ b/seed/go-sdk/webhooks/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/websocket-bearer-auth/internal/retrier.go b/seed/go-sdk/websocket-bearer-auth/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/websocket-bearer-auth/internal/retrier.go +++ b/seed/go-sdk/websocket-bearer-auth/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/websocket-bearer-auth/internal/retrier_test.go b/seed/go-sdk/websocket-bearer-auth/internal/retrier_test.go index 8278a5867411..78cc1853317a 100644 --- a/seed/go-sdk/websocket-bearer-auth/internal/retrier_test.go +++ b/seed/go-sdk/websocket-bearer-auth/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/websocket-inferred-auth/internal/retrier.go b/seed/go-sdk/websocket-inferred-auth/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/websocket-inferred-auth/internal/retrier.go +++ b/seed/go-sdk/websocket-inferred-auth/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/websocket-inferred-auth/internal/retrier_test.go b/seed/go-sdk/websocket-inferred-auth/internal/retrier_test.go index 9964c8294072..99a144d35cd0 100644 --- a/seed/go-sdk/websocket-inferred-auth/internal/retrier_test.go +++ b/seed/go-sdk/websocket-inferred-auth/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/websocket-multi-url/internal/retrier.go b/seed/go-sdk/websocket-multi-url/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/websocket-multi-url/internal/retrier.go +++ b/seed/go-sdk/websocket-multi-url/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/websocket-multi-url/internal/retrier_test.go b/seed/go-sdk/websocket-multi-url/internal/retrier_test.go index e308ad11adba..c14bab9e62f9 100644 --- a/seed/go-sdk/websocket-multi-url/internal/retrier_test.go +++ b/seed/go-sdk/websocket-multi-url/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/websocket/internal/retrier.go b/seed/go-sdk/websocket/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/websocket/internal/retrier.go +++ b/seed/go-sdk/websocket/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/websocket/internal/retrier_test.go b/seed/go-sdk/websocket/internal/retrier_test.go index 6fb9411a5ac8..ce76446a984a 100644 --- a/seed/go-sdk/websocket/internal/retrier_test.go +++ b/seed/go-sdk/websocket/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/x-fern-default/apply-query-defaults-on-nil-request/internal/retrier.go b/seed/go-sdk/x-fern-default/apply-query-defaults-on-nil-request/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/x-fern-default/apply-query-defaults-on-nil-request/internal/retrier.go +++ b/seed/go-sdk/x-fern-default/apply-query-defaults-on-nil-request/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/x-fern-default/apply-query-defaults-on-nil-request/internal/retrier_test.go b/seed/go-sdk/x-fern-default/apply-query-defaults-on-nil-request/internal/retrier_test.go index 3e942018a167..60f6d092313a 100644 --- a/seed/go-sdk/x-fern-default/apply-query-defaults-on-nil-request/internal/retrier_test.go +++ b/seed/go-sdk/x-fern-default/apply-query-defaults-on-nil-request/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/x-fern-default/no-custom-config/internal/retrier.go b/seed/go-sdk/x-fern-default/no-custom-config/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/x-fern-default/no-custom-config/internal/retrier.go +++ b/seed/go-sdk/x-fern-default/no-custom-config/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/x-fern-default/no-custom-config/internal/retrier_test.go b/seed/go-sdk/x-fern-default/no-custom-config/internal/retrier_test.go index 3e942018a167..60f6d092313a 100644 --- a/seed/go-sdk/x-fern-default/no-custom-config/internal/retrier_test.go +++ b/seed/go-sdk/x-fern-default/no-custom-config/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string diff --git a/seed/go-sdk/x-fern-global-parameters/internal/retrier.go b/seed/go-sdk/x-fern-global-parameters/internal/retrier.go index caeedc22956a..16baf1488818 100644 --- a/seed/go-sdk/x-fern-global-parameters/internal/retrier.go +++ b/seed/go-sdk/x-fern-global-parameters/internal/retrier.go @@ -145,13 +145,21 @@ func (r *Retrier) run( } if r.shouldRetry(response) { - defer func() { _ = response.Body.Close() }() - delay, err := r.retryDelay(response, retryAttempt) if err != nil { + _ = response.Body.Close() return nil, err } + // If the context's deadline would elapse before the backoff completes, return + // the response instead of sleeping through it. The caller then sees why the + // request actually failed (e.g. a 429) rather than a context deadline error. + if deadline, ok := request.Context().Deadline(); ok && time.Until(deadline) < delay { + return response, nil + } + + defer func() { _ = response.Body.Close() }() + if err := sleepWithContext(request.Context(), delay); err != nil { return nil, err } diff --git a/seed/go-sdk/x-fern-global-parameters/internal/retrier_test.go b/seed/go-sdk/x-fern-global-parameters/internal/retrier_test.go index 5a48a33cf5d3..b1ad92320ae8 100644 --- a/seed/go-sdk/x-fern-global-parameters/internal/retrier_test.go +++ b/seed/go-sdk/x-fern-global-parameters/internal/retrier_test.go @@ -295,6 +295,9 @@ func TestRetryWithRequestBody(t *testing.T) { assert.Equal(t, expectedBody, requestBodies[1], "Second request body should match expected (retry should re-send body)") } +// An explicit cancel (e.g. Ctrl-C wired to cancel()) must interrupt the backoff +// wait. There is no deadline here, so the retrier cannot know the wait is futile +// up front -- it has to be woken by the context. func TestRetryWaitIsInterruptedByContext(t *testing.T) { var requestCount int server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -308,7 +311,11 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { Client: server.Client(), }) - ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + time.Sleep(100 * time.Millisecond) + cancel() + }() defer cancel() start := time.Now() @@ -323,11 +330,88 @@ func TestRetryWaitIsInterruptedByContext(t *testing.T) { ) elapsed := time.Since(start) - assert.ErrorIs(t, err, context.DeadlineExceeded) - assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context expired") + assert.ErrorIs(t, err, context.Canceled) + assert.Equal(t, 1, requestCount, "Expected the retry to be abandoned once the context was cancelled") assert.Less(t, elapsed, time.Second, "Expected the backoff to be interrupted by the context, took %v", elapsed) } +// When the caller's deadline would elapse before the backoff finishes, sleeping +// only guarantees a context error. Return the response instead so the caller +// learns why the request actually failed. +func TestRetryReturnsResponseWhenDeadlineShorterThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + w.Header().Set("Retry-After", "5") + w.WriteHeader(http.StatusTooManyRequests) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + + start := time.Now() + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + MaxAttempts: 3, + }, + ) + elapsed := time.Since(start) + + // The caller gets the 429, not context.DeadlineExceeded. + var apiError *core.APIError + require.ErrorAs(t, err, &apiError) + assert.Equal(t, http.StatusTooManyRequests, apiError.StatusCode) + assert.NotErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, 1, requestCount, "Expected no retry once the deadline was known to be too short") + assert.Less(t, elapsed, time.Second, "Expected to return immediately rather than sleep, took %v", elapsed) +} + +// A deadline with room for the backoff must still retry as normal. +func TestRetryProceedsWhenDeadlineLongerThanBackoff(t *testing.T) { + var requestCount int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requestCount++ + if requestCount == 1 { + w.Header().Set("Retry-After", "1") + w.WriteHeader(http.StatusTooManyRequests) + return + } + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("{}")) + })) + defer server.Close() + + caller := NewCaller(&CallerParams{ + Client: server.Client(), + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + _, err := caller.Call( + ctx, + &CallParams{ + URL: server.URL, + Method: http.MethodGet, + Request: &InternalTestRequest{}, + Response: &InternalTestResponse{}, + MaxAttempts: 3, + }, + ) + + assert.NoError(t, err) + assert.Equal(t, 2, requestCount, "Expected the retry to proceed when the deadline allows it") +} + func TestDisableRetries(t *testing.T) { tests := []struct { name string