From 8abae076d193676d983ac58a245d28233f9124d7 Mon Sep 17 00:00:00 2001
From: Anthony Romano <anthony.romano@coreos.com>
Date: Tue, 19 Jul 2016 12:33:33 -0700
Subject: [PATCH] rpctypes, clientv3: retry RPC on EtcdStopped

Fixes #5983
---
 clientv3/client.go                     | 9 +++++++--
 clientv3/client_test.go                | 9 ++++++++-
 etcdserver/api/v3rpc/rpctypes/error.go | 3 +++
 3 files changed, 18 insertions(+), 3 deletions(-)

diff --git a/clientv3/client.go b/clientv3/client.go
index 7a8fe28b77e..66d5a3c0f96 100644
--- a/clientv3/client.go
+++ b/clientv3/client.go
@@ -275,8 +275,13 @@ func isHaltErr(ctx context.Context, err error) bool {
 	if err == nil {
 		return false
 	}
-	return strings.HasPrefix(grpc.ErrorDesc(err), "etcdserver: ") ||
-		strings.Contains(err.Error(), grpc.ErrClientConnClosing.Error())
+	eErr := rpctypes.Error(err)
+	if _, ok := eErr.(rpctypes.EtcdError); ok {
+		return eErr != rpctypes.ErrStopped && eErr != rpctypes.ErrNoLeader
+	}
+	// treat etcdserver errors not recognized by the client as halting
+	return strings.Contains(err.Error(), grpc.ErrClientConnClosing.Error()) ||
+		strings.Contains(err.Error(), "etcdserver:")
 }
 
 func toErr(ctx context.Context, err error) error {
diff --git a/clientv3/client_test.go b/clientv3/client_test.go
index db42037f299..63ff8146e18 100644
--- a/clientv3/client_test.go
+++ b/clientv3/client_test.go
@@ -19,6 +19,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/coreos/etcd/etcdserver"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 )
@@ -57,7 +58,13 @@ func TestDialTimeout(t *testing.T) {
 
 func TestIsHaltErr(t *testing.T) {
 	if !isHaltErr(nil, fmt.Errorf("etcdserver: some etcdserver error")) {
-		t.Errorf(`error prefixed with "etcdserver: " should be Halted`)
+		t.Errorf(`error prefixed with "etcdserver: " should be Halted by default`)
+	}
+	if isHaltErr(nil, etcdserver.ErrStopped) {
+		t.Errorf("error %v should not halt", etcdserver.ErrStopped)
+	}
+	if isHaltErr(nil, etcdserver.ErrNoLeader) {
+		t.Errorf("error %v should not halt", etcdserver.ErrNoLeader)
 	}
 	ctx, cancel := context.WithCancel(context.TODO())
 	if isHaltErr(ctx, nil) {
diff --git a/etcdserver/api/v3rpc/rpctypes/error.go b/etcdserver/api/v3rpc/rpctypes/error.go
index 3f4510e61f9..b41d1c83354 100644
--- a/etcdserver/api/v3rpc/rpctypes/error.go
+++ b/etcdserver/api/v3rpc/rpctypes/error.go
@@ -53,6 +53,7 @@ var (
 
 	ErrGRPCNoLeader   = grpc.Errorf(codes.Unavailable, "etcdserver: no leader")
 	ErrGRPCNotCapable = grpc.Errorf(codes.Unavailable, "etcdserver: not capable")
+	ErrGRPCStopped    = grpc.Errorf(codes.Unavailable, "etcdserver: server stopped")
 
 	errStringToError = map[string]error{
 		grpc.ErrorDesc(ErrGRPCEmptyKey):     ErrGRPCEmptyKey,
@@ -87,6 +88,7 @@ var (
 
 		grpc.ErrorDesc(ErrGRPCNoLeader):   ErrGRPCNoLeader,
 		grpc.ErrorDesc(ErrGRPCNotCapable): ErrGRPCNotCapable,
+		grpc.ErrorDesc(ErrGRPCStopped):    ErrGRPCStopped,
 	}
 
 	// client-side error
@@ -122,6 +124,7 @@ var (
 
 	ErrNoLeader   = Error(ErrGRPCNoLeader)
 	ErrNotCapable = Error(ErrGRPCNotCapable)
+	ErrStopped    = Error(ErrGRPCStopped)
 )
 
 // EtcdError defines gRPC server errors.