Implement the retriable resiliency pattern

README.md

@@ -6,3 +6,4 @@ Resiliency patterns for golang. Currently implemented are:
 - semaphore pattern (in the `semaphore` directory)
 - deadline/timeout pattern (in the `deadline` directory)
 - batching pattern (in the `batcher` directory)
+- retriable pattern (in the `retrier` directory)

retrier/README.md

@@ -0,0 +1,25 @@
+retrier
+=======
+
+[![Build Status](https://travis-ci.org/eapache/go-resiliency.svg?branch=master)](https://travis-ci.org/eapache/go-resiliency)
+[![GoDoc](https://godoc.org/github.com/eapache/go-resiliency/retrier?status.svg)](https://godoc.org/github.com/eapache/go-resiliency/retrier)
+
+The retriable resiliency pattern for golang.
+
+Creating a retrier takes two parameters:
+- the times to back-off between retries (and implicitly the number of times to
+  retry)
+- the classifier that determines which errors to retry
+
+```go
+r := retrier.New(retrier.ConstantBackoff(3, 100*time.Millisecond), nil)
+
+err := r.Run(func() error {
+	// do some work
+	return nil
+})
+
+if err != nil {
+	// handle the case where the work failed three times
+}
+```

retrier/backoffs.go

@@ -0,0 +1,24 @@
+package retrier
+
+import "time"
+
+// ConstantBackoff generates a simple back-off strategy of retrying 'n' times, and waiting 'amount' time after each one.
+func ConstantBackoff(n int, amount time.Duration) []time.Duration {
+	ret := make([]time.Duration, n)
+	for i := range ret {
+		ret[i] = amount
+	}
+	return ret
+}
+
+// ExponentialBackoff generates a simple back-off strategy of retrying 'n' times, and doubling the amount of
+// time waited after each one.
+func ExponentialBackoff(n int, initialAmount time.Duration) []time.Duration {
+	ret := make([]time.Duration, n)
+	next := initialAmount
+	for i := range ret {
+		ret[i] = next
+		next *= 2
+	}
+	return ret
+}

retrier/backoffs_test.go

@@ -0,0 +1,55 @@
+package retrier
+
+import (
+	"testing"
+	"time"
+)
+
+func TestConstantBackoff(t *testing.T) {
+	b := ConstantBackoff(1, 10*time.Millisecond)
+	if len(b) != 1 {
+		t.Error("incorrect length")
+	}
+	for i := range b {
+		if b[i] != 10*time.Millisecond {
+			t.Error("incorrect value at", i)
+		}
+	}
+
+	b = ConstantBackoff(10, 250*time.Hour)
+	if len(b) != 10 {
+		t.Error("incorrect length")
+	}
+	for i := range b {
+		if b[i] != 250*time.Hour {
+			t.Error("incorrect value at", i)
+		}
+	}
+}
+
+func TestExponentialBackoff(t *testing.T) {
+	b := ExponentialBackoff(1, 10*time.Millisecond)
+	if len(b) != 1 {
+		t.Error("incorrect length")
+	}
+	if b[0] != 10*time.Millisecond {
+		t.Error("incorrect value")
+	}
+
+	b = ExponentialBackoff(4, 1*time.Minute)
+	if len(b) != 4 {
+		t.Error("incorrect length")
+	}
+	if b[0] != 1*time.Minute {
+		t.Error("incorrect value")
+	}
+	if b[1] != 2*time.Minute {
+		t.Error("incorrect value")
+	}
+	if b[2] != 4*time.Minute {
+		t.Error("incorrect value")
+	}
+	if b[3] != 8*time.Minute {
+		t.Error("incorrect value")
+	}
+}

retrier/classifier.go

@@ -0,0 +1,66 @@
+package retrier
+
+// Action is the type returned by a Classifier to indicate how the Retrier should proceed.
+type Action int
+
+const (
+	Succeed Action = iota // Succeed indicates the Retrier should treat this value as a success.
+	Fail                  // Fail indicates the Retrier should treat this value as a hard failure and not retry.
+	Retry                 // Retry indicates the Retrier should treat this value as a soft failure and retry.
+)
+
+// Classifier is the interface implemented by anything that can classify Errors for a Retrier.
+type Classifier interface {
+	Classify(error) Action
+}
+
+// DefaultClassifier classifies errors in the simplest way possible. If
+// the error is nil, it returns Succeed, otherwise it returns Retry.
+type DefaultClassifier struct{}
+
+// Classify implements the Classifier interface.
+func (c DefaultClassifier) Classify(err error) Action {
+	if err == nil {
+		return Succeed
+	}
+
+	return Retry
+}
+
+// WhitelistClassifier classifies errors based on a whitelist. If the error is nil, it
+// returns Succeed; if the error is in the whitelist, it returns Retry; otherwise, it returns Fail.
+type WhitelistClassifier []error
+
+// Classify implements the Classifier interface.
+func (list WhitelistClassifier) Classify(err error) Action {
+	if err == nil {
+		return Succeed
+	}
+
+	for _, pass := range list {
+		if err == pass {
+			return Retry
+		}
+	}
+
+	return Fail
+}
+
+// BlacklistClassifier classifies errors based on a blacklist. If the error is nil, it
+// returns Succeed; if the error is in the blacklist, it returns Fail; otherwise, it returns Retry.
+type BlacklistClassifier []error
+
+// Classify implements the Classifier interface.
+func (list BlacklistClassifier) Classify(err error) Action {
+	if err == nil {
+		return Succeed
+	}
+
+	for _, pass := range list {
+		if err == pass {
+			return Fail
+		}
+	}
+
+	return Retry
+}

retrier/classifier_test.go

@@ -0,0 +1,66 @@
+package retrier
+
+import (
+	"errors"
+	"testing"
+)
+
+var (
+	errFoo = errors.New("FOO")
+	errBar = errors.New("BAR")
+	errBaz = errors.New("BAZ")
+)
+
+func TestDefaultClassifier(t *testing.T) {
+	c := DefaultClassifier{}
+
+	if c.Classify(nil) != Succeed {
+		t.Error("default misclassified nil")
+	}
+
+	if c.Classify(errFoo) != Retry {
+		t.Error("default misclassified foo")
+	}
+	if c.Classify(errBar) != Retry {
+		t.Error("default misclassified bar")
+	}
+	if c.Classify(errBaz) != Retry {
+		t.Error("default misclassified baz")
+	}
+}
+
+func TestWhitelistClassifier(t *testing.T) {
+	c := WhitelistClassifier{errFoo, errBar}
+
+	if c.Classify(nil) != Succeed {
+		t.Error("whitelist misclassified nil")
+	}
+
+	if c.Classify(errFoo) != Retry {
+		t.Error("whitelist misclassified foo")
+	}
+	if c.Classify(errBar) != Retry {
+		t.Error("whitelist misclassified bar")
+	}
+	if c.Classify(errBaz) != Fail {
+		t.Error("whitelist misclassified baz")
+	}
+}
+
+func TestBlacklistClassifier(t *testing.T) {
+	c := BlacklistClassifier{errBar}
+
+	if c.Classify(nil) != Succeed {
+		t.Error("blacklist misclassified nil")
+	}
+
+	if c.Classify(errFoo) != Retry {
+		t.Error("blacklist misclassified foo")
+	}
+	if c.Classify(errBar) != Fail {
+		t.Error("blacklist misclassified bar")
+	}
+	if c.Classify(errBaz) != Retry {
+		t.Error("blacklist misclassified baz")
+	}
+}

retrier/retrier.go

@@ -0,0 +1,49 @@
+// Package retrier implements the "retriable" resiliency pattern for Go.
+package retrier
+
+import "time"
+
+// Retrier implements the "retriable" resiliency pattern, abstracting out the process of retrying a failed action
+// a certain number of times with an optional back-off between each retry.
+type Retrier struct {
+	backoff []time.Duration
+	class   Classifier
+}
+
+// New constructs a Retrier with the given backoff pattern and classifier. The length of the backoff pattern
+// indicates how many times an action will be retried, and the value at each index indicates the amount of time
+// waited before each subsequent retry. The classifier is used to determine which errors should be retried and
+// which should cause the retrier to fail fast. The DefaultClassifier is used if nil is passed.
+func New(backoff []time.Duration, class Classifier) *Retrier {
+	if class == nil {
+		class = DefaultClassifier{}
+	}
+
+	return &Retrier{
+		backoff: backoff,
+		class:   class,
+	}
+}
+
+// Run executes the given work function, then classifies its return value based on the classifier used
+// to construct the Retrier. If the result is Succeed or Fail, the return value of the work function is
+// returned to the caller. If the result is Retry, then Run sleeps according to the its backoff policy
+// before retrying. If the total number of retries is exceeded then the return value of the work function
+// is returned to the caller regardless.
+func (r *Retrier) Run(work func() error) error {
+	retries := 0
+	for {
+		ret := work()
+
+		switch r.class.Classify(ret) {
+		case Succeed, Fail:
+			return ret
+		case Retry:
+			if retries >= len(r.backoff) {
+				return ret
+			}
+			time.Sleep(r.backoff[retries])
+			retries++
+		}
+	}
+}

retrier/retrier_test.go

@@ -0,0 +1,88 @@
+package retrier
+
+import (
+	"testing"
+	"time"
+)
+
+var i int
+
+func genWork(returns []error) func() error {
+	i = 0
+	return func() error {
+		i++
+		if i > len(returns) {
+			return nil
+		}
+		return returns[i-1]
+	}
+}
+
+func TestRetrier(t *testing.T) {
+	r := New([]time.Duration{0, 10 * time.Millisecond}, WhitelistClassifier{errFoo})
+
+	err := r.Run(genWork([]error{errFoo, errFoo}))
+	if err != nil {
+		t.Error(err)
+	}
+	if i != 3 {
+		t.Error("run wrong number of times")
+	}
+
+	err = r.Run(genWork([]error{errFoo, errBar}))
+	if err != errBar {
+		t.Error(err)
+	}
+	if i != 2 {
+		t.Error("run wrong number of times")
+	}
+
+	err = r.Run(genWork([]error{errBar, errBaz}))
+	if err != errBar {
+		t.Error(err)
+	}
+	if i != 1 {
+		t.Error("run wrong number of times")
+	}
+}
+
+func TestRetrierNone(t *testing.T) {
+	r := New(nil, nil)
+
+	i = 0
+	err := r.Run(func() error {
+		i++
+		return errFoo
+	})
+	if err != errFoo {
+		t.Error(err)
+	}
+	if i != 1 {
+		t.Error("run wrong number of times")
+	}
+
+	i = 0
+	err = r.Run(func() error {
+		i++
+		return nil
+	})
+	if err != nil {
+		t.Error(err)
+	}
+	if i != 1 {
+		t.Error("run wrong number of times")
+	}
+}
+
+func ExampleRetrier() {
+	r := New(ConstantBackoff(3, 100*time.Millisecond), nil)
+
+	err := r.Run(func() error {
+		// do some work
+		return nil
+	})
+
+	if err != nil {
+		// handle the case where the work failed three times
+	}
+}