functional-tester/tester: add random delay failure cases

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

tools/functional-tester/tester/cluster.go

@@ -96,6 +96,16 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
 			clus.Members[i].Etcd.WALDir = filepath.Join(clus.Members[i].Etcd.DataDir, "member", "wal")
 		}
 
+		if clus.Members[i].Etcd.HeartbeatIntervalMs == 0 {
+			return nil, fmt.Errorf("'--heartbeat-interval' cannot be 0 (got %+v)", clus.Members[i].Etcd)
+		}
+		if clus.Members[i].Etcd.ElectionTimeoutMs == 0 {
+			return nil, fmt.Errorf("'--election-timeout' cannot be 0 (got %+v)", clus.Members[i].Etcd)
+		}
+		if int64(clus.Tester.DelayLatencyMs) <= clus.Members[i].Etcd.ElectionTimeoutMs {
+			return nil, fmt.Errorf("delay latency %d ms must be greater than election timeout %d ms", clus.Tester.DelayLatencyMs, clus.Members[i].Etcd.ElectionTimeoutMs)
+		}
+
 		port := ""
 		listenClientPorts := make([]string, len(clus.Members))
 		for i, u := range clus.Members[i].Etcd.ListenClientURLs {
@@ -160,6 +170,10 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
 		}
 	}
 
+	if clus.Tester.DelayLatencyMs <= clus.Tester.DelayLatencyMsRv {
+		return nil, fmt.Errorf("delay latency %d ms must be greater than delay latency random variable %d ms", clus.Tester.DelayLatencyMs, clus.Tester.DelayLatencyMsRv)
+	}
+
 	for _, v := range clus.Tester.FailureCases {
 		if _, ok := rpcpb.FailureCase_value[v]; !ok {
 			return nil, fmt.Errorf("%q is not defined in 'rpcpb.FailureCase_value'", v)

tools/functional-tester/tester/cluster_test.go

@@ -138,11 +138,17 @@ func Test_newCluster(t *testing.T) {
 				"BLACKHOLE_PEER_PORT_TX_RX_QUORUM",
 				"BLACKHOLE_PEER_PORT_TX_RX_ALL",
 				"DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER",
+				"RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER",
 				"DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT",
+				"RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT",
 				"DELAY_PEER_PORT_TX_RX_LEADER",
+				"RANDOM_DELAY_PEER_PORT_TX_RX_LEADER",
 				"DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT",
+				"RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT",
 				"DELAY_PEER_PORT_TX_RX_QUORUM",
+				"RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM",
 				"DELAY_PEER_PORT_TX_RX_ALL",
+				"RANDOM_DELAY_PEER_PORT_TX_RX_ALL",
 				"NO_FAIL_WITH_STRESS",
 				"NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS",
 			},

tools/functional-tester/tester/local-test.yaml

@@ -105,11 +105,17 @@ tester-config:
   - BLACKHOLE_PEER_PORT_TX_RX_QUORUM
   - BLACKHOLE_PEER_PORT_TX_RX_ALL
   - DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER
+  - RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER
   - DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT
+  - RANDOM_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER_UNTIL_TRIGGER_SNAPSHOT
   - DELAY_PEER_PORT_TX_RX_LEADER
+  - RANDOM_DELAY_PEER_PORT_TX_RX_LEADER
   - DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT
+  - RANDOM_DELAY_PEER_PORT_TX_RX_LEADER_UNTIL_TRIGGER_SNAPSHOT
   - DELAY_PEER_PORT_TX_RX_QUORUM
+  - RANDOM_DELAY_PEER_PORT_TX_RX_QUORUM
   - DELAY_PEER_PORT_TX_RX_ALL
+  - RANDOM_DELAY_PEER_PORT_TX_RX_ALL
   - NO_FAIL_WITH_STRESS
   - NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS