clientv3: fix race in KV reconnection logic

clientv3/kv.go

@@ -183,14 +183,18 @@ func (kv *kv) Do(ctx context.Context, op Op) (OpResponse, error) {
 }
 
 func (kv *kv) switchRemote(prevErr error) error {
+	// Usually it's a bad idea to lock on network i/o but here it's OK
+	// since the link is down and new requests can't be processed anyway.
+	// Likewise, if connecting stalls, closing the Client can break the
+	// lock via context cancelation.
+	kv.mu.Lock()
+	defer kv.mu.Unlock()
+
 	newConn, err := kv.c.retryConnection(kv.conn, prevErr)
 	if err != nil {
 		return err
 	}
 
-	kv.mu.Lock()
-	defer kv.mu.Unlock()
-
 	kv.conn = newConn
 	kv.remote = pb.NewKVClient(kv.conn)
 	return nil