rpctypes, clientv3: retry RPC on EtcdStopped

Fixes #5983

clientv3/client.go

@@ -275,8 +275,13 @@ func isHaltErr(ctx context.Context, err error) bool {
 	if err == nil {
 		return false
 	}
-	return strings.HasPrefix(grpc.ErrorDesc(err), "etcdserver: ") ||
-		strings.Contains(err.Error(), grpc.ErrClientConnClosing.Error())
+	eErr := rpctypes.Error(err)
+	if _, ok := eErr.(rpctypes.EtcdError); ok {
+		return eErr != rpctypes.ErrStopped && eErr != rpctypes.ErrNoLeader
+	}
+	// treat etcdserver errors not recognized by the client as halting
+	return strings.Contains(err.Error(), grpc.ErrClientConnClosing.Error()) ||
+		strings.Contains(err.Error(), "etcdserver:")
 }
 
 func toErr(ctx context.Context, err error) error {

clientv3/client_test.go

@@ -19,6 +19,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/coreos/etcd/etcdserver"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 )
@@ -57,7 +58,13 @@ func TestDialTimeout(t *testing.T) {
 
 func TestIsHaltErr(t *testing.T) {
 	if !isHaltErr(nil, fmt.Errorf("etcdserver: some etcdserver error")) {
-		t.Errorf(`error prefixed with "etcdserver: " should be Halted`)
+		t.Errorf(`error prefixed with "etcdserver: " should be Halted by default`)
+	}
+	if isHaltErr(nil, etcdserver.ErrStopped) {
+		t.Errorf("error %v should not halt", etcdserver.ErrStopped)
+	}
+	if isHaltErr(nil, etcdserver.ErrNoLeader) {
+		t.Errorf("error %v should not halt", etcdserver.ErrNoLeader)
 	}
 	ctx, cancel := context.WithCancel(context.TODO())
 	if isHaltErr(ctx, nil) {

etcdserver/api/v3rpc/rpctypes/error.go

@@ -53,6 +53,7 @@ var (
 
 	ErrGRPCNoLeader   = grpc.Errorf(codes.Unavailable, "etcdserver: no leader")
 	ErrGRPCNotCapable = grpc.Errorf(codes.Unavailable, "etcdserver: not capable")
+	ErrGRPCStopped    = grpc.Errorf(codes.Unavailable, "etcdserver: server stopped")
 
 	errStringToError = map[string]error{
 		grpc.ErrorDesc(ErrGRPCEmptyKey):     ErrGRPCEmptyKey,
@@ -87,6 +88,7 @@ var (
 
 		grpc.ErrorDesc(ErrGRPCNoLeader):   ErrGRPCNoLeader,
 		grpc.ErrorDesc(ErrGRPCNotCapable): ErrGRPCNotCapable,
+		grpc.ErrorDesc(ErrGRPCStopped):    ErrGRPCStopped,
 	}
 
 	// client-side error
@@ -122,6 +124,7 @@ var (
 
 	ErrNoLeader   = Error(ErrGRPCNoLeader)
 	ErrNotCapable = Error(ErrGRPCNotCapable)
+	ErrStopped    = Error(ErrGRPCStopped)
 )
 
 // EtcdError defines gRPC server errors.