Skip to content

request id and retry logic #21

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
code7a wants to merge 1 commit into
base: v2
Choose a base branch
from
Open

request id and retry logic #21

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 38 additions & 6 deletions http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,18 @@ type APIResponse struct {
Request *http.Request
ReqBody string
Warnings []string
// RequestID is the PCE's X-Request-Id response header, used by Illumio support to trace a request server-side.
RequestID string
// RetryLog holds one entry per failed attempt made before this response, oldest first. Empty if the call succeeded on the first try.
RetryLog []RetryAttempt
}

// RetryAttempt records a single failed attempt that was retried by httpReq.
type RetryAttempt struct {
Attempt int
StatusCode int
RequestID string
Err string
}

// Unexported struct for handling the asyncResults
Expand Down Expand Up @@ -172,6 +184,7 @@ func (p *PCE) httpSetup(action, apiURL string, body []byte, async bool, headers
response.StatusCode = resp.StatusCode
response.Header = resp.Header
response.Request = resp.Request
response.RequestID = resp.Header.Get("X-Request-Id")

// Check for a 200 response code
if strconv.Itoa(resp.StatusCode)[0:1] != "2" {
Expand Down Expand Up @@ -256,23 +269,42 @@ func (p *PCE) httpReq(action, apiURL string, body []byte, async bool, headers ma
// Make initial http call
api, err := p.httpSetup(action, apiURL, body, async, headers)
retry := 0
var retryLog []RetryAttempt

// If the status code is 429, try 3 times
for api.StatusCode == 429 {
// If we have already tried 3 times, exit
// Retry on 429 (rate limit), 5xx (server error), or a transport-level error with no response at all
for isRetryableFailure(api, err) {
// If we have already tried 6 times, exit
if retry > 6 {
return api, errors.New("received 6 429 errors with 30 second pauses between attempts")
api.RetryLog = retryLog
return api, fmt.Errorf("received 6 errors with 30 second pauses between attempts - last status code %d", api.StatusCode)
}
// Record this failed attempt before retrying
errMsg := ""
if err != nil {
errMsg = err.Error()
}
// Increment the retry counter and sleep for 30 seconds
retry++
retryLog = append(retryLog, RetryAttempt{Attempt: retry, StatusCode: api.StatusCode, RequestID: api.RequestID, Err: errMsg})
// Sleep for 30 seconds
time.Sleep(30 * time.Second)
// Retry the API call
api, err = p.httpSetup(action, apiURL, body, async, headers)
}
// Return once response code isn't 429
// Attach the retry history, if any, to the final response
api.RetryLog = retryLog
// Return once response is successful or a non-retryable failure
return api, err
}

// isRetryableFailure determines if an API response/error should trigger a retry:
// 429, 5xx, or a transport-level error with no HTTP response at all.
func isRetryableFailure(api APIResponse, err error) bool {
if api.StatusCode == 429 || api.StatusCode >= 500 {
return true
}
return api.StatusCode == 0 && err != nil
}

// cleanFQDN cleans up the provided PCE FQDN in case of common errors
func (p *PCE) cleanFQDN() string {
// Remove trailing slash if included
Expand Down