gophercloud · jtopjian · Jan 23, 2021 · Jan 15, 2021 · Jan 22, 2021
diff --git a/.zuul/playbooks/gophercloud-acceptance-test-ironic/run.yaml b/.zuul/playbooks/gophercloud-acceptance-test-ironic/run.yaml
@@ -1,7 +1,8 @@
 - hosts: all
   become: yes
   roles:
-    - config-golang
+    - role: config-golang
+      go_version: '1.15'
     - clone-devstack-gate-to-workspace
     - role: create-devstack-local-conf
       enable_services:

diff --git a/.zuul/playbooks/gophercloud-acceptance-test/run.yaml b/.zuul/playbooks/gophercloud-acceptance-test/run.yaml
@@ -1,7 +1,8 @@
 - hosts: all
   become: yes
   roles:
-    - config-golang
+    - role: config-golang
+      go_version: '1.15'
     - clone-devstack-gate-to-workspace
     - role: create-devstack-local-conf
       enable_services:

diff --git a/.zuul/playbooks/gophercloud-unittest/run.yaml b/.zuul/playbooks/gophercloud-unittest/run.yaml
@@ -1,7 +1,8 @@
 - hosts: all
   become: yes
   roles:
-    - config-golang
+    - role: config-golang
+      go_version: '1.15'
   tasks:
     - name: Run unit tests with gophercloud
       shell:

diff --git a/doc.go b/doc.go
@@ -106,5 +106,43 @@ intermediary processing on each page, you can use the AllPages method:
 This top-level package contains utility functions and data types that are used
 throughout the provider and service packages. Of particular note for end users
 are the AuthOptions and EndpointOpts structs.
+
+An example retry backoff function, which respects the 429 HTTP response code and a "Retry-After" header:
+
+	endpoint := "http://localhost:5000"
+	provider, err := openstack.NewClient(endpoint)
+	if err != nil {
+		panic(err)
+	}
+	provider.MaxBackoffRetries = 3 // max three retries
+	provider.RetryBackoffFunc = func(ctx context.Context, respErr *ErrUnexpectedResponseCode, e error, retries uint) error {
+		retryAfter := respErr.ResponseHeader.Get("Retry-After")
+		if retryAfter == "" {
+			return e
+		}
+
+		var sleep time.Duration
+
+		// Parse delay seconds or HTTP date
+		if v, err := strconv.ParseUint(retryAfter, 10, 32); err == nil {
+			sleep = time.Duration(v) * time.Second
+		} else if v, err := time.Parse(http.TimeFormat, retryAfter); err != nil {
+			return e
+		} else {
+			sleep = v.UTC().Sub(time.Now().UTC())
+		}
+
+		if ctx != nil {
+			select {
+			case <-time.After(sleep):
+			case <-ctx.Done():
+			}
+		} else {
+			time.Sleep(sleep)
+		}
+
+		return nil
+	}
+
 */
 package gophercloud
diff --git a/provider_client.go b/provider_client.go
@@ -13,7 +13,10 @@ import (
 )
 
 // DefaultUserAgent is the default User-Agent string set in the request header.
-const DefaultUserAgent = "gophercloud/2.0.0"
+const (
+	DefaultUserAgent         = "gophercloud/2.0.0"
+	DefaultMaxBackoffRetries = 60
+)
 
 // UserAgent represents a User-Agent header.
 type UserAgent struct {
@@ -80,6 +83,12 @@ type ProviderClient struct {
 	// Context is the context passed to the HTTP request.
 	Context context.Context
 
+	// Retry backoff func
+	RetryBackoffFunc func(context.Context, *ErrUnexpectedResponseCode, error, uint) error
+
+	// MaxBackoffRetries set the maximum number of backoffs. When not set, defaults to DefaultMaxBackoffRetries
+	MaxBackoffRetries uint
+
 	// mut is a mutex for the client. It protects read and write access to client attributes such as getting
 	// and setting the TokenID.
 	mut *sync.RWMutex
@@ -323,6 +332,8 @@ type requestState struct {
 	// reauthenticate, but keep getting 401 responses with the fresh token, reauthenticating some more
 	// will just get us into an infinite loop.
 	hasReauthenticated bool
+	// Retry-After backoff counter, increments during each backoff call
+	retries uint
 }
 
 var applicationJSON = "application/json"
@@ -503,6 +514,24 @@ func (client *ProviderClient) doRequest(method, url string, options *RequestOpts
 			if error429er, ok := errType.(Err429er); ok {
 				err = error429er.Error429(respErr)
 			}
+
+			maxTries := client.MaxBackoffRetries
+			if maxTries == 0 {
+				maxTries = DefaultMaxBackoffRetries
+			}
+
+			if f := client.RetryBackoffFunc; f != nil && state.retries < maxTries {
+				var e error
+
+				state.retries = state.retries + 1
+				e = f(client.Context, &respErr, err, state.retries)
+
+				if e != nil {
+					return resp, e
+				}
+
+				return client.doRequest(method, url, options, state)
+			}
 		case http.StatusInternalServerError:
 			err = ErrDefault500{respErr}
 			if error500er, ok := errType.(Err500er); ok {

diff --git a/testing/provider_client_test.go b/testing/provider_client_test.go
@@ -7,6 +7,7 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -448,3 +449,187 @@ func TestRequestConnectionClose(t *testing.T) {
 
 	th.AssertEquals(t, int64(iter), connections)
 }
+
+type testRetryFunc func(context.Context, *gophercloud.ErrUnexpectedResponseCode, error, uint) error
+
+func retryTest(retryCounter *uint, t *testing.T) testRetryFunc {
+	return func(ctx context.Context, respErr *gophercloud.ErrUnexpectedResponseCode, e error, retries uint) error {
+		retryAfter := respErr.ResponseHeader.Get("Retry-After")
+		if retryAfter == "" {
+			return e
+		}
+
+		var sleep time.Duration
+
+		// Parse delay seconds or HTTP date
+		if v, err := strconv.ParseUint(retryAfter, 10, 32); err == nil {
+			sleep = time.Duration(v) * time.Second
+		} else if v, err := time.Parse(http.TimeFormat, retryAfter); err != nil {
+			return e
+		} else {
+			sleep = v.UTC().Sub(time.Now().UTC())
+		}
+
+		if ctx != nil {
+			t.Logf("Context sleeping for %d milliseconds", sleep.Milliseconds())
+			select {
+			case <-time.After(sleep):
+				t.Log("sleep is over")
+			case <-ctx.Done():
+				t.Log("context exceeded")
+				return e
+			}
+		} else {
+			t.Logf("Sleeping for %d milliseconds", sleep.Milliseconds())
+			time.Sleep(sleep)
+			t.Log("sleep is over")
+		}
+
+		*retryCounter = *retryCounter + 1
+
+		return nil
+	}
+}
+
+func TestRequestRetry(t *testing.T) {
+	var retryCounter uint
+
+	p := &gophercloud.ProviderClient{}
+	p.UseTokenLock()
+	p.SetToken(client.TokenID)
+	p.MaxBackoffRetries = 3
+
+	p.RetryBackoffFunc = retryTest(&retryCounter, t)
+
+	th.SetupHTTP()
+	defer th.TeardownHTTP()
+
+	th.Mux.HandleFunc("/route", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Retry-After", "1")
+
+		//always reply 429
+		http.Error(w, "retry later", http.StatusTooManyRequests)
+	})
+
+	_, err := p.Request("GET", th.Endpoint()+"/route", &gophercloud.RequestOpts{})
+	if err == nil {
+		t.Fatal("expecting error, got nil")
+	}
+	th.AssertEquals(t, retryCounter, p.MaxBackoffRetries)
+}
+
+func TestRequestRetryHTTPDate(t *testing.T) {
+	var retryCounter uint
+
+	p := &gophercloud.ProviderClient{}
+	p.UseTokenLock()
+	p.SetToken(client.TokenID)
+	p.MaxBackoffRetries = 3
+
+	p.RetryBackoffFunc = retryTest(&retryCounter, t)
+
+	th.SetupHTTP()
+	defer th.TeardownHTTP()
+
+	th.Mux.HandleFunc("/route", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Retry-After", time.Now().Add(1*time.Second).UTC().Format(http.TimeFormat))
+
+		//always reply 429
+		http.Error(w, "retry later", http.StatusTooManyRequests)
+	})
+
+	_, err := p.Request("GET", th.Endpoint()+"/route", &gophercloud.RequestOpts{})
+	if err == nil {
+		t.Fatal("expecting error, got nil")
+	}
+	th.AssertEquals(t, retryCounter, p.MaxBackoffRetries)
+}
+
+func TestRequestRetryError(t *testing.T) {
+	var retryCounter uint
+
+	p := &gophercloud.ProviderClient{}
+	p.UseTokenLock()
+	p.SetToken(client.TokenID)
+	p.MaxBackoffRetries = 3
+
+	p.RetryBackoffFunc = retryTest(&retryCounter, t)
+
+	th.SetupHTTP()
+	defer th.TeardownHTTP()
+
+	th.Mux.HandleFunc("/route", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Retry-After", "foo bar")
+
+		//always reply 429
+		http.Error(w, "retry later", http.StatusTooManyRequests)
+	})
+
+	_, err := p.Request("GET", th.Endpoint()+"/route", &gophercloud.RequestOpts{})
+	if err == nil {
+		t.Fatal("expecting error, got nil")
+	}
+	th.AssertEquals(t, retryCounter, uint(0))
+}
+
+func TestRequestRetrySuccess(t *testing.T) {
+	var retryCounter uint
+
+	p := &gophercloud.ProviderClient{}
+	p.UseTokenLock()
+	p.SetToken(client.TokenID)
+	p.MaxBackoffRetries = 3
+
+	p.RetryBackoffFunc = retryTest(&retryCounter, t)
+
+	th.SetupHTTP()
+	defer th.TeardownHTTP()
+
+	th.Mux.HandleFunc("/route", func(w http.ResponseWriter, r *http.Request) {
+		//always reply 200
+		http.Error(w, "retry later", http.StatusOK)
+	})
+
+	_, err := p.Request("GET", th.Endpoint()+"/route", &gophercloud.RequestOpts{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	th.AssertEquals(t, retryCounter, uint(0))
+}
+
+func TestRequestRetryContext(t *testing.T) {
+	var retryCounter uint
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		sleep := 2.5 * 1000 * time.Millisecond
+		time.Sleep(sleep)
+		cancel()
+	}()
+
+	p := &gophercloud.ProviderClient{
+		Context: ctx,
+	}
+	p.UseTokenLock()
+	p.SetToken(client.TokenID)
+	p.MaxBackoffRetries = 3
+
+	p.RetryBackoffFunc = retryTest(&retryCounter, t)
+
+	th.SetupHTTP()
+	defer th.TeardownHTTP()
+
+	th.Mux.HandleFunc("/route", func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Retry-After", "1")
+
+		//always reply 429
+		http.Error(w, "retry later", http.StatusTooManyRequests)
+	})
+
+	_, err := p.Request("GET", th.Endpoint()+"/route", &gophercloud.RequestOpts{})
+	if err == nil {
+		t.Fatal("expecting error, got nil")
+	}
+	t.Logf("retryCounter: %d, p.MaxBackoffRetries: %d", retryCounter, p.MaxBackoffRetries-1)
+	th.AssertEquals(t, retryCounter, p.MaxBackoffRetries-1)
+}