functional/tester: fix "failureQuorum"

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

functional/tester/failure.go

@@ -143,11 +143,15 @@ func (f *failureLeader) FailureCase() rpcpb.FailureCase {
 	return f.failureCase
 }
 
-type failureQuorum failureByFunc
+type failureQuorum struct {
+	failureByFunc
+	injected map[int]struct{}
+}
 
 func (f *failureQuorum) Inject(clus *Cluster) error {
-	for i := range killMap(len(clus.Members), clus.rd) {
-		if err := f.injectMember(clus, i); err != nil {
+	f.injected = pickQuorum(len(clus.Members))
+	for idx := range f.injected {
+		if err := f.injectMember(clus, idx); err != nil {
 			return err
 		}
 	}
@@ -155,8 +159,8 @@ func (f *failureQuorum) Inject(clus *Cluster) error {
 }
 
 func (f *failureQuorum) Recover(clus *Cluster) error {
-	for i := range killMap(len(clus.Members), clus.rd) {
-		if err := f.recoverMember(clus, i); err != nil {
+	for idx := range f.injected {
+		if err := f.recoverMember(clus, idx); err != nil {
 			return err
 		}
 	}
@@ -174,16 +178,15 @@ func (f *failureQuorum) FailureCase() rpcpb.FailureCase {
 	return f.failureCase
 }
 
-func killMap(size int, seed int) map[int]bool {
-	m := make(map[int]bool)
-	r := rand.New(rand.NewSource(int64(seed)))
-	majority := size/2 + 1
-	for {
-		m[r.Intn(size)] = true
-		if len(m) >= majority {
-			return m
-		}
+func pickQuorum(size int) (picked map[int]struct{}) {
+	picked = make(map[int]struct{})
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	quorum := size/2 + 1
+	for len(picked) < quorum {
+		idx := r.Intn(size)
+		picked[idx] = struct{}{}
 	}
+	return picked
 }
 
 type failureAll failureByFunc

functional/tester/failure_case_failpoints.go

@@ -114,10 +114,13 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure)
 				lead: -1,
 			},
 			&failureQuorum{
-				desc:          fmt.Sprintf("failpoint %q (quorum: %q)", fp, fcmd),
-				failureCase:   rpcpb.FailureCase_FAILPOINTS,
-				injectMember:  inject,
-				recoverMember: recov,
+				failureByFunc: failureByFunc{
+					desc:          fmt.Sprintf("failpoint %q (quorum: %q)", fp, fcmd),
+					failureCase:   rpcpb.FailureCase_FAILPOINTS,
+					injectMember:  inject,
+					recoverMember: recov,
+				},
+				injected: make(map[int]struct{}),
 			},
 			&failureAll{
 				desc:          fmt.Sprintf("failpoint %q (all: %q)", fp, fcmd),

functional/tester/failure_case_network_blackhole.go

@@ -78,9 +78,12 @@ func new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT() F
 
 func new_FailureCase_BLACKHOLE_PEER_PORT_TX_RX_QUORUM(clus *Cluster) Failure {
 	f := &failureQuorum{
-		failureCase:   rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_QUORUM,
-		injectMember:  inject_BLACKHOLE_PEER_PORT_TX_RX,
-		recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX,
+		failureByFunc: failureByFunc{
+			failureCase:   rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_QUORUM,
+			injectMember:  inject_BLACKHOLE_PEER_PORT_TX_RX,
+			recoverMember: recover_BLACKHOLE_PEER_PORT_TX_RX,
+		},
+		injected: make(map[int]struct{}),
 	}
 	return &failureDelay{
 		Failure:       f,

functional/tester/failure_case_network_delay.go

@@ -120,9 +120,12 @@ func new_FailureCase_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus *C
 
 func new_FailureCase_DELAY_PEER_PORT_TX_RX_QUORUM(clus *Cluster, random bool) Failure {
 	f := &failureQuorum{
-		failureCase:   rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_QUORUM,
-		injectMember:  inject_DELAY_PEER_PORT_TX_RX,
-		recoverMember: recover_DELAY_PEER_PORT_TX_RX,
+		failureByFunc: failureByFunc{
+			failureCase:   rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_QUORUM,
+			injectMember:  inject_DELAY_PEER_PORT_TX_RX,
+			recoverMember: recover_DELAY_PEER_PORT_TX_RX,
+		},
+		injected: make(map[int]struct{}),
 	}
 	clus.Tester.UpdatedDelayLatencyMs = clus.Tester.DelayLatencyMs
 	if random {

functional/tester/failure_case_sigterm.go

@@ -66,9 +66,12 @@ func new_FailureCase_SIGTERM_LEADER_UNTIL_TRIGGER_SNAPSHOT(clus *Cluster) Failur
 
 func new_FailureCase_SIGTERM_QUORUM(clus *Cluster) Failure {
 	f := &failureQuorum{
-		failureCase:   rpcpb.FailureCase_SIGTERM_QUORUM,
-		injectMember:  inject_SIGTERM_ETCD,
-		recoverMember: recover_SIGTERM_ETCD,
+		failureByFunc: failureByFunc{
+			failureCase:   rpcpb.FailureCase_SIGTERM_QUORUM,
+			injectMember:  inject_SIGTERM_ETCD,
+			recoverMember: recover_SIGTERM_ETCD,
+		},
+		injected: make(map[int]struct{}),
 	}
 	return &failureDelay{
 		Failure:       f,