functional-tester/tester: add "failure-delay-ms" for configurable triggerElectionDur

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

tools/functional-tester/tester/cluster.go

@@ -276,11 +276,11 @@ func (clus *Cluster) updateFailures() {
 		case "KILL_ALL":
 			clus.failures = append(clus.failures, newFailureKillAll())
 		case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER":
-			clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower())
+			clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower(clus))
 		case "BLACKHOLE_PEER_PORT_TX_RX_LEADER":
-			clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader())
+			clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader(clus))
 		case "BLACKHOLE_PEER_PORT_TX_RX_ALL":
-			clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll())
+			clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll(clus))
 		case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER":
 			clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxOneFollower(clus))
 		case "DELAY_PEER_PORT_TX_RX_LEADER":

tools/functional-tester/tester/cluster_test.go

@@ -134,6 +134,7 @@ func Test_newCluster(t *testing.T) {
 				"NO_FAIL_WITH_STRESS",
 				"NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS",
 			},
+			FailureDelayMs:          7000,
 			FailureShuffle:          true,
 			FailpointCommands:       []string{`panic("etcd-tester")`},
 			RunnerExecPath:          "/etcd-runner",

tools/functional-tester/tester/failure_case_failpoints.go

@@ -59,7 +59,7 @@ func failpointFailures(clus *Cluster) (ret []Failure, err error) {
 			} else {
 				fpFails[i] = &failureDelay{
 					Failure:       fpf,
-					delayDuration: 3 * time.Second,
+					delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
 				}
 			}
 		}

tools/functional-tester/tester/failure_case_network_blackhole.go

@@ -14,7 +14,11 @@
 
 package tester
 
-import "github.com/coreos/etcd/tools/functional-tester/rpcpb"
+import (
+	"time"
+
+	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
+)
 
 func injectBlackholePeerPortTxRx(clus *Cluster, idx int) error {
 	return clus.sendOperation(idx, rpcpb.Operation_BlackholePeerPortTxRx)
@@ -24,7 +28,7 @@ func recoverBlackholePeerPortTxRx(clus *Cluster, idx int) error {
 	return clus.sendOperation(idx, rpcpb.Operation_UnblackholePeerPortTxRx)
 }
 
-func newFailureBlackholePeerPortTxRxOneFollower() Failure {
+func newFailureBlackholePeerPortTxRxOneFollower(clus *Cluster) Failure {
 	ff := failureByFunc{
 		failureCase:   rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER,
 		injectMember:  injectBlackholePeerPortTxRx,
@@ -33,11 +37,11 @@ func newFailureBlackholePeerPortTxRxOneFollower() Failure {
 	f := &failureFollower{ff, -1, -1}
 	return &failureDelay{
 		Failure:       f,
-		delayDuration: triggerElectionDur,
+		delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
 	}
 }
 
-func newFailureBlackholePeerPortTxRxLeader() Failure {
+func newFailureBlackholePeerPortTxRxLeader(clus *Cluster) Failure {
 	ff := failureByFunc{
 		failureCase:   rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER,
 		injectMember:  injectBlackholePeerPortTxRx,
@@ -46,11 +50,11 @@ func newFailureBlackholePeerPortTxRxLeader() Failure {
 	f := &failureLeader{ff, -1, -1}
 	return &failureDelay{
 		Failure:       f,
-		delayDuration: triggerElectionDur,
+		delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
 	}
 }
 
-func newFailureBlackholePeerPortTxRxAll() Failure {
+func newFailureBlackholePeerPortTxRxAll(clus *Cluster) Failure {
 	f := &failureAll{
 		failureCase:   rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL,
 		injectMember:  injectBlackholePeerPortTxRx,
@@ -58,6 +62,6 @@ func newFailureBlackholePeerPortTxRxAll() Failure {
 	}
 	return &failureDelay{
 		Failure:       f,
-		delayDuration: triggerElectionDur,
+		delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
 	}
 }

tools/functional-tester/tester/failure_case_network_slow.go

@@ -21,9 +21,6 @@ import (
 )
 
 const (
-	// delay duration to trigger leader election (default election timeout 1s)
-	triggerElectionDur = 5 * time.Second
-
 	// Wait more when it recovers from slow network, because network layer
 	// needs extra time to propagate traffic control (tc command) change.
 	// Otherwise, we get different hash values from the previous revision.
@@ -50,7 +47,7 @@ func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster) Failure {
 	f := &failureFollower{ff, -1, -1}
 	return &failureDelay{
 		Failure:       f,
-		delayDuration: triggerElectionDur,
+		delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
 	}
 }
 
@@ -63,7 +60,7 @@ func newFailureDelayPeerPortTxRxLeader(clus *Cluster) Failure {
 	f := &failureLeader{ff, -1, -1}
 	return &failureDelay{
 		Failure:       f,
-		delayDuration: triggerElectionDur,
+		delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
 	}
 }
 
@@ -75,6 +72,6 @@ func newFailureDelayPeerPortTxRxAll(clus *Cluster) Failure {
 	}
 	return &failureDelay{
 		Failure:       f,
-		delayDuration: triggerElectionDur,
+		delayDuration: time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond,
 	}
 }

tools/functional-tester/tester/local-test.yaml

@@ -100,6 +100,7 @@ tester-config:
   - NO_FAIL_WITH_STRESS
   - NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS
 
+  failure-delay-ms: 7000
   failure-shuffle: true
   failpoint-commands:
   - panic("etcd-tester")