Browse Source

integration: retry remove in TestDecreaseClusterSize

Cluster may go through a second leader election if test machine is
overloaded. Retry remove until it passes without error.

Fixes #8225
Anthony Romano 8 years ago
parent
commit
59d232adf9
2 changed files with 12 additions and 3 deletions
  1. 3 2
      integration/cluster.go
  2. 9 1
      integration/cluster_test.go

+ 3 - 2
integration/cluster.go

@@ -303,10 +303,11 @@ func (c *cluster) removeMember(t *testing.T, id uint64) error {
 	cc := MustNewHTTPClient(t, c.URLs(), c.cfg.ClientTLS)
 	ma := client.NewMembersAPI(cc)
 	ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
-	if err := ma.Remove(ctx, types.ID(id).String()); err != nil {
+	err := ma.Remove(ctx, types.ID(id).String())
+	cancel()
+	if err != nil {
 		return err
 	}
-	cancel()
 	newMembers := make([]*member, 0)
 	for _, m := range c.Members {
 		if uint64(m.s.ID()) != id {

+ 9 - 1
integration/cluster_test.go

@@ -151,7 +151,15 @@ func testDecreaseClusterSize(t *testing.T, size int) {
 	// TODO: remove the last but one member
 	for i := 0; i < size-1; i++ {
 		id := c.Members[len(c.Members)-1].s.ID()
-		c.RemoveMember(t, uint64(id))
+		// may hit second leader election on slow machines
+		if err := c.removeMember(t, uint64(id)); err != nil {
+			if strings.Contains(err.Error(), "no leader") {
+				t.Logf("got leader error (%v)", err)
+				i--
+				continue
+			}
+			t.Fatal(err)
+		}
 		c.waitLeader(t, c.Members)
 	}
 	clusterMustProgress(t, c.Members)