Add rate limiting to incident.io client and improve retry logic

client/client.go

@@ -8,15 +8,18 @@ import (
 	"time"
 
 	"github.com/deepmap/oapi-codegen/pkg/securityprovider"
-	"github.com/hashicorp/go-cleanhttp"
 	"github.com/hashicorp/go-retryablehttp"
 	"github.com/pkg/errors"
 	"golang.org/x/time/rate"
 )
 
 const (
-	rpm              = 600
-	bucketsPerMinute = 60
+	rpm = 600
+
+	// last retry will wait 64 seconds
+	maxRetries   = 7
+	minRetryWait = 1 * time.Second
+	maxRetryWait = 65 * time.Second
 )
 
 type RateLimitedClient struct {
@@ -28,7 +31,7 @@ func (c *RateLimitedClient) Do(req *http.Request) (*http.Response, error) {
 	if c.rateLimiter == nil {
 		return nil, errors.New("rate limiter not set")
 	}
-	ctx := context.Background()
+	ctx := req.Context()
 	err := c.rateLimiter.Wait(ctx)
 	if err != nil {
 		return nil, err
@@ -40,21 +43,44 @@ func (c *RateLimitedClient) Do(req *http.Request) (*http.Response, error) {
 	return resp, nil
 }
 
+func attentiveBackoff(min, max time.Duration, attemptNum int, resp *http.Response) time.Duration {
+	// Retry for rate limits and server errors.
+	if resp != nil && resp.StatusCode == http.StatusTooManyRequests {
+		// Check for a 'Retry-After' header.
+		retryAfter := resp.Header.Get("Retry-After")
+		if retryAfter != "" {
+			if retryAfterSeconds, err := time.ParseDuration(retryAfter + "s"); err == nil {
+				return retryAfterSeconds
+			}
+		}
+
+	}
+	// otherwise use the default backoff
+	return retryablehttp.DefaultBackoff(min, max, attemptNum, resp)
+}
+
 func New(ctx context.Context, apiKey, apiEndpoint, version string, opts ...ClientOption) (*ClientWithResponses, error) {
 	bearerTokenProvider, bearerTokenProviderErr := securityprovider.NewSecurityProviderBearerToken(apiKey)
 	if bearerTokenProviderErr != nil {
 		return nil, bearerTokenProviderErr
 	}
 
 	retryClient := retryablehttp.NewClient()
-	retryClient.RetryMax = 3
+
+	retryClient.RetryMax = maxRetries
+	retryClient.RetryWaitMin = minRetryWait
+	retryClient.RetryWaitMax = maxRetryWait
+	retryClient.Backoff = attentiveBackoff
 
 	base := retryClient.StandardClient()
 
 	// The generated client won't turn validation errors into actual errors, so we do this
 	// inside of a generic middleware.
-	base.Transport = Wrap(cleanhttp.DefaultTransport(), func(req *http.Request, next http.RoundTripper) (*http.Response, error) {
+	base.Transport = Wrap(base.Transport, func(req *http.Request, next http.RoundTripper) (*http.Response, error) {
 		resp, err := next.RoundTrip(req)
+		if err != nil {
+			return nil, err
+		}
 		if err == nil && resp.StatusCode > 299 {
 			data, err := io.ReadAll(resp.Body)
 			if err != nil {
@@ -67,9 +93,10 @@ func New(ctx context.Context, apiKey, apiEndpoint, version string, opts ...Clien
 		return resp, err
 	})
 
+	// Adding a naive rate limiter to prevent us from using the API keys entire allowance
 	rlClient := &RateLimitedClient{
 		client:      base,
-		rateLimiter: rate.NewLimiter(rate.Every(time.Minute/bucketsPerMinute), rpm/bucketsPerMinute),
+		rateLimiter: rate.NewLimiter(rpm/60, 1),
 	}
 
 	clientOpts := append([]ClientOption{

development.md

@@ -11,10 +11,13 @@ go test ./...
 To build the binary, run `make`: this will place a built version of the tool in
 the `bin` directory. If you work for incident.io and have a local instance of
 the app running, then you can point it to your local environment using the
-`--api-key` flag, or an environment variable:
+following environment variable:
 
 ```
+export INCIDENT_API_KEY="inc_development_YOUR_API_KEY"
 export INCIDENT_ENDPOINT="http://localhost:3001/api/public"
 ```
 
+To run it with a debugger attached you can edit the settings in `launch.json`
+
 [go]: https://go.dev/doc/install

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 	github.com/alecthomas/kingpin/v2 v2.3.2
 	github.com/bmatcuk/doublestar/v4 v4.6.0
+	github.com/davecgh/go-spew v1.1.1
 	github.com/deepmap/oapi-codegen v1.12.4
 	github.com/fatih/color v1.12.0
 	github.com/ghodss/yaml v1.0.0