Merge pull request #9528 from gyuho/no-op

functional-tester: add liveness mode

test

@@ -234,7 +234,7 @@ function cov_pass {
 	mkdir -p "$COVERDIR"
 
 	# run code coverage for unit and integration tests
-	GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 15m"
+	GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 20m"
 	# shellcheck disable=SC2206
 	GOCOVFLAGS=($GOCOVFLAGS)
 	failed=""
@@ -292,7 +292,7 @@ function e2e_pass {
 	# expectation could be different
 	USERTIMEOUT=""
 	if [ -z "${TIMEOUT}" ]; then
-		USERTIMEOUT="15m"
+		USERTIMEOUT="20m"
 	else
 		USERTIMEOUT="${TIMEOUT}"
 	fi
@@ -303,9 +303,9 @@ function e2e_pass {
 function integration_e2e_pass {
 	echo "Running integration and e2e tests..."
 
-	go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" &
+	go test -timeout 20m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" &
 	e2epid="$!"
-	go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" &
+	go test -timeout 20m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" &
 	intpid="$!"
 	wait $e2epid
 	wait $intpid
@@ -315,7 +315,7 @@ function integration_e2e_pass {
 function grpcproxy_pass {
 	go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/integration"
 	go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration"
-	go test -timeout 15m -v -tags cluster_proxy "$@" "${REPO_PATH}/e2e"
+	go test -timeout 20m -v -tags cluster_proxy "$@" "${REPO_PATH}/e2e"
 }
 
 function release_pass {

tools/functional-tester/rpcpb/rpc.pb.go

@@ -104,9 +104,14 @@ const (
 	FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER     FailureCase = 9
 	FailureCase_DELAY_PEER_PORT_TX_RX_LEADER           FailureCase = 10
 	FailureCase_DELAY_PEER_PORT_TX_RX_ALL              FailureCase = 11
-	FailureCase_FAILPOINTS                             FailureCase = 100
-	FailureCase_NO_FAIL                                FailureCase = 200
-	FailureCase_EXTERNAL                               FailureCase = 300
+	// NO_FAIL_WITH_STRESS runs no-op failure injection for specified period
+	// while stressers are still sending requests.
+	FailureCase_NO_FAIL_WITH_STRESS FailureCase = 100
+	// NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS runs no-op failure injection
+	// with all stressers stopped.
+	FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS FailureCase = 101
+	FailureCase_FAILPOINTS                          FailureCase = 200
+	FailureCase_EXTERNAL                            FailureCase = 300
 )
 
 var FailureCase_name = map[int32]string{
@@ -122,8 +127,9 @@ var FailureCase_name = map[int32]string{
 	9:   "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER",
 	10:  "DELAY_PEER_PORT_TX_RX_LEADER",
 	11:  "DELAY_PEER_PORT_TX_RX_ALL",
-	100: "FAILPOINTS",
-	200: "NO_FAIL",
+	100: "NO_FAIL_WITH_STRESS",
+	101: "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS",
+	200: "FAILPOINTS",
 	300: "EXTERNAL",
 }
 var FailureCase_value = map[string]int32{
@@ -139,8 +145,9 @@ var FailureCase_value = map[string]int32{
 	"DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER":     9,
 	"DELAY_PEER_PORT_TX_RX_LEADER":           10,
 	"DELAY_PEER_PORT_TX_RX_ALL":              11,
-	"FAILPOINTS":                             100,
-	"NO_FAIL":                                200,
+	"NO_FAIL_WITH_STRESS":                    100,
+	"NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS":    101,
+	"FAILPOINTS":                             200,
 	"EXTERNAL":                               300,
 }
 
@@ -154,30 +161,27 @@ type StressType int32
 const (
 	StressType_KV                StressType = 0
 	StressType_LEASE             StressType = 1
-	StressType_NO_STRESS         StressType = 2
-	StressType_ELECTION_RUNNER   StressType = 3
-	StressType_WATCH_RUNNER      StressType = 4
-	StressType_LOCK_RACER_RUNNER StressType = 5
-	StressType_LEASE_RUNNER      StressType = 6
+	StressType_ELECTION_RUNNER   StressType = 2
+	StressType_WATCH_RUNNER      StressType = 3
+	StressType_LOCK_RACER_RUNNER StressType = 4
+	StressType_LEASE_RUNNER      StressType = 5
 )
 
 var StressType_name = map[int32]string{
 	0: "KV",
 	1: "LEASE",
-	2: "NO_STRESS",
-	3: "ELECTION_RUNNER",
-	4: "WATCH_RUNNER",
-	5: "LOCK_RACER_RUNNER",
-	6: "LEASE_RUNNER",
+	2: "ELECTION_RUNNER",
+	3: "WATCH_RUNNER",
+	4: "LOCK_RACER_RUNNER",
+	5: "LEASE_RUNNER",
 }
 var StressType_value = map[string]int32{
 	"KV":                0,
 	"LEASE":             1,
-	"NO_STRESS":         2,
-	"ELECTION_RUNNER":   3,
-	"WATCH_RUNNER":      4,
-	"LOCK_RACER_RUNNER": 5,
-	"LEASE_RUNNER":      6,
+	"ELECTION_RUNNER":   2,
+	"WATCH_RUNNER":      3,
+	"LOCK_RACER_RUNNER": 4,
+	"LEASE_RUNNER":      5,
 }
 
 func (x StressType) String() string {
@@ -255,12 +259,14 @@ type Tester struct {
 	EnablePprof bool `protobuf:"varint,24,opt,name=EnablePprof,proto3" json:"EnablePprof,omitempty" yaml:"enable-pprof"`
 	// FailureCases is the selected test cases to schedule.
 	// If empty, run all failure cases.
-	// TODO: support no-op
 	FailureCases []string `protobuf:"bytes,31,rep,name=FailureCases" json:"FailureCases,omitempty" yaml:"failure-cases"`
+	// FailureDelayMs is the delay duration after failure is injected.
+	// Useful when triggering snapshot or no-op failure cases.
+	FailureDelayMs uint32 `protobuf:"varint,32,opt,name=FailureDelayMs,proto3" json:"FailureDelayMs,omitempty" yaml:"failure-delay-ms"`
 	// FailureShuffle is true to randomize failure injecting order.
-	FailureShuffle bool `protobuf:"varint,32,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"`
+	FailureShuffle bool `protobuf:"varint,33,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"`
 	// FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)).
-	FailpointCommands []string `protobuf:"bytes,33,rep,name=FailpointCommands" json:"FailpointCommands,omitempty" yaml:"failpoint-commands"`
+	FailpointCommands []string `protobuf:"bytes,34,rep,name=FailpointCommands" json:"FailpointCommands,omitempty" yaml:"failpoint-commands"`
 	// RunnerExecPath is a path of etcd-runner binary.
 	RunnerExecPath string `protobuf:"bytes,41,opt,name=RunnerExecPath,proto3" json:"RunnerExecPath,omitempty" yaml:"runner-exec-path"`
 	// ExternalExecPath is a path of script for enabling/disabling an external fault injector.
@@ -280,8 +286,11 @@ type Tester struct {
 	StressKeySuffixRangeTxn int32 `protobuf:"varint,105,opt,name=StressKeySuffixRangeTxn,proto3" json:"StressKeySuffixRangeTxn,omitempty" yaml:"stress-key-suffix-range-txn"`
 	// StressKeyTxnOps is the number of operations per a transaction (max 64).
 	StressKeyTxnOps int32 `protobuf:"varint,106,opt,name=StressKeyTxnOps,proto3" json:"StressKeyTxnOps,omitempty" yaml:"stress-key-txn-ops"`
+	// StressClients is the number of concurrent stressing clients
+	// with "one" shared TCP connection.
+	StressClients int32 `protobuf:"varint,201,opt,name=StressClients,proto3" json:"StressClients,omitempty" yaml:"stress-clients"`
 	// StressQPS is the maximum number of stresser requests per second.
-	StressQPS int32 `protobuf:"varint,107,opt,name=StressQPS,proto3" json:"StressQPS,omitempty" yaml:"stress-qps"`
+	StressQPS int32 `protobuf:"varint,202,opt,name=StressQPS,proto3" json:"StressQPS,omitempty" yaml:"stress-qps"`
 }
 
 func (m *Tester) Reset()                    { *m = Tester{} }
@@ -774,11 +783,18 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) {
 			i += copy(dAtA[i:], s)
 		}
 	}
-	if m.FailureShuffle {
+	if m.FailureDelayMs != 0 {
 		dAtA[i] = 0x80
 		i++
 		dAtA[i] = 0x2
 		i++
+		i = encodeVarintRpc(dAtA, i, uint64(m.FailureDelayMs))
+	}
+	if m.FailureShuffle {
+		dAtA[i] = 0x88
+		i++
+		dAtA[i] = 0x2
+		i++
 		if m.FailureShuffle {
 			dAtA[i] = 1
 		} else {
@@ -788,7 +804,7 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) {
 	}
 	if len(m.FailpointCommands) > 0 {
 		for _, s := range m.FailpointCommands {
-			dAtA[i] = 0x8a
+			dAtA[i] = 0x92
 			i++
 			dAtA[i] = 0x2
 			i++
@@ -871,10 +887,17 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) {
 		i++
 		i = encodeVarintRpc(dAtA, i, uint64(m.StressKeyTxnOps))
 	}
+	if m.StressClients != 0 {
+		dAtA[i] = 0xc8
+		i++
+		dAtA[i] = 0xc
+		i++
+		i = encodeVarintRpc(dAtA, i, uint64(m.StressClients))
+	}
 	if m.StressQPS != 0 {
-		dAtA[i] = 0xd8
+		dAtA[i] = 0xd0
 		i++
-		dAtA[i] = 0x6
+		dAtA[i] = 0xc
 		i++
 		i = encodeVarintRpc(dAtA, i, uint64(m.StressQPS))
 	}
@@ -1111,6 +1134,9 @@ func (m *Tester) Size() (n int) {
 			n += 2 + l + sovRpc(uint64(l))
 		}
 	}
+	if m.FailureDelayMs != 0 {
+		n += 2 + sovRpc(uint64(m.FailureDelayMs))
+	}
 	if m.FailureShuffle {
 		n += 3
 	}
@@ -1149,6 +1175,9 @@ func (m *Tester) Size() (n int) {
 	if m.StressKeyTxnOps != 0 {
 		n += 2 + sovRpc(uint64(m.StressKeyTxnOps))
 	}
+	if m.StressClients != 0 {
+		n += 2 + sovRpc(uint64(m.StressClients))
+	}
 	if m.StressQPS != 0 {
 		n += 2 + sovRpc(uint64(m.StressQPS))
 	}
@@ -2167,6 +2196,25 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
 			m.FailureCases = append(m.FailureCases, string(dAtA[iNdEx:postIndex]))
 			iNdEx = postIndex
 		case 32:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field FailureDelayMs", wireType)
+			}
+			m.FailureDelayMs = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowRpc
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.FailureDelayMs |= (uint32(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 33:
 			if wireType != 0 {
 				return fmt.Errorf("proto: wrong wireType = %d for field FailureShuffle", wireType)
 			}
@@ -2186,7 +2234,7 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
 				}
 			}
 			m.FailureShuffle = bool(v != 0)
-		case 33:
+		case 34:
 			if wireType != 2 {
 				return fmt.Errorf("proto: wrong wireType = %d for field FailpointCommands", wireType)
 			}
@@ -2397,7 +2445,26 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
 					break
 				}
 			}
-		case 107:
+		case 201:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field StressClients", wireType)
+			}
+			m.StressClients = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowRpc
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				m.StressClients |= (int32(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+		case 202:
 			if wireType != 0 {
 				return fmt.Errorf("proto: wrong wireType = %d for field StressQPS", wireType)
 			}
@@ -2779,119 +2846,122 @@ var (
 func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptorRpc) }
 
 var fileDescriptorRpc = []byte{
-	// 1813 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x57, 0x51, 0x73, 0xda, 0xca,
-	0x15, 0x36, 0xb6, 0xc1, 0x66, 0x6d, 0x6c, 0xdd, 0xb5, 0x1d, 0x2b, 0x4e, 0x62, 0x71, 0xd5, 0xde,
-	0x8c, 0xeb, 0x19, 0x39, 0x6d, 0xee, 0x4c, 0x67, 0x7a, 0xe7, 0x76, 0x52, 0xc0, 0xca, 0x8d, 0x6b,
-	0x05, 0xc8, 0x82, 0x93, 0xf4, 0x89, 0x0a, 0x69, 0x01, 0xd5, 0x42, 0x52, 0x56, 0x8b, 0x2f, 0xdc,
-	0x87, 0xbe, 0x76, 0xfa, 0x0f, 0xda, 0xff, 0xd0, 0xfe, 0x8f, 0xb4, 0xbd, 0x0f, 0xfd, 0x05, 0xb4,
-	0x4d, 0xff, 0x01, 0x2f, 0x7d, 0xed, 0xec, 0xae, 0x80, 0x15, 0xe0, 0xf6, 0x8d, 0x3d, 0xe7, 0xfb,
-	0xbe, 0x3d, 0x7b, 0x74, 0xce, 0xd9, 0x05, 0xec, 0x93, 0xc8, 0x89, 0xda, 0xcf, 0x48, 0xe4, 0x5c,
-	0x44, 0x24, 0xa4, 0x21, 0xcc, 0x72, 0xc3, 0x89, 0xd1, 0xf5, 0x68, 0x6f, 0xd0, 0xbe, 0x70, 0xc2,
-	0xfe, 0xb3, 0x6e, 0xd8, 0x0d, 0x9f, 0x71, 0x6f, 0x7b, 0xd0, 0xe1, 0x2b, 0xbe, 0xe0, 0xbf, 0x04,
-	0x4b, 0xff, 0x7e, 0x0b, 0x6c, 0x9a, 0xd4, 0x71, 0xe1, 0x0f, 0xc0, 0x66, 0xd5, 0xee, 0x63, 0x35,
-	0x53, 0xcc, 0x9c, 0xe5, 0xcb, 0xfb, 0x93, 0xb1, 0xb6, 0x33, 0xb2, 0xfb, 0xfe, 0x57, 0x7a, 0x60,
-	0xf7, 0xb1, 0x8e, 0xb8, 0x13, 0x1a, 0x60, 0xeb, 0xd2, 0xa6, 0xf6, 0xa5, 0x47, 0xd4, 0x75, 0x8e,
-	0x3b, 0x98, 0x8c, 0xb5, 0x7d, 0x81, 0x73, 0x6d, 0x6a, 0x1b, 0xae, 0x47, 0x74, 0x34, 0xc5, 0xc0,
-	0x73, 0x90, 0x7b, 0x57, 0xb2, 0x18, 0x7a, 0x83, 0xa3, 0xe1, 0x64, 0xac, 0xed, 0x09, 0xf4, 0xb7,
-	0xb6, 0x2f, 0xc0, 0x09, 0x02, 0x5e, 0x01, 0xc5, 0xf2, 0x62, 0x8a, 0x83, 0x8a, 0xef, 0xe1, 0x80,
-	0xde, 0x20, 0x2b, 0x56, 0x37, 0x8b, 0x1b, 0x67, 0xf9, 0xf2, 0x93, 0xc9, 0x58, 0x7b, 0x28, 0x58,
-	0x3e, 0x47, 0x18, 0x0e, 0x87, 0x18, 0x03, 0xe2, 0xc7, 0x3a, 0x5a, 0xa2, 0x41, 0x04, 0x0e, 0x4a,
-	0xee, 0x1d, 0x26, 0xd4, 0x8b, 0xb1, 0xa4, 0x96, 0xe5, 0x6a, 0xc5, 0xc9, 0x58, 0x7b, 0x2c, 0xd4,
-	0xec, 0x29, 0x28, 0x2d, 0xb8, 0x8a, 0x0c, 0x2b, 0x60, 0x4f, 0xec, 0x53, 0xc7, 0x98, 0x70, 0xb9,
-	0x1c, 0x97, 0x7b, 0x34, 0x19, 0x6b, 0xc7, 0xa9, 0xe0, 0x22, 0x8c, 0x49, 0xa2, 0xb4, 0x40, 0x81,
-	0x6d, 0xa0, 0x5e, 0x05, 0x1e, 0xf5, 0x6c, 0x7f, 0xb6, 0xc5, 0x4c, 0x6e, 0x8b, 0xcb, 0x3d, 0x9d,
-	0x8c, 0x35, 0x5d, 0xc8, 0x79, 0x02, 0x69, 0xcc, 0xa3, 0x94, 0x94, 0xef, 0xd5, 0x81, 0x65, 0xb0,
-	0x97, 0xf8, 0x2a, 0xfe, 0x20, 0xa6, 0x98, 0xa8, 0xdb, 0x3c, 0xf7, 0x27, 0x93, 0xb1, 0xf6, 0x20,
-	0xad, 0xec, 0x08, 0x80, 0x8e, 0x16, 0x18, 0x2c, 0x81, 0x69, 0x4b, 0x83, 0xda, 0x14, 0xab, 0x79,
-	0x2e, 0x24, 0x25, 0x70, 0x41, 0xc8, 0x88, 0x19, 0x4c, 0x47, 0xab, 0xc8, 0xcb, 0x9a, 0xcd, 0xf0,
-	0x16, 0x07, 0x2a, 0xf8, 0x7f, 0x9a, 0x94, 0xc1, 0x96, 0x34, 0x39, 0x19, 0xbe, 0x00, 0x85, 0x46,
-	0x60, 0x47, 0x71, 0x2f, 0xa4, 0x95, 0x70, 0x10, 0x50, 0x75, 0xa7, 0x98, 0x39, 0xdb, 0x28, 0x3f,
-	0x9c, 0x8c, 0xb5, 0x23, 0xa1, 0x16, 0x27, 0x6e, 0xc3, 0x61, 0x7e, 0x1d, 0xa5, 0xf1, 0xd0, 0x02,
-	0x9f, 0xbd, 0x19, 0x84, 0xd4, 0x2e, 0xdb, 0xce, 0x2d, 0x0e, 0xdc, 0xf2, 0x88, 0xe2, 0x58, 0xdd,
-	0xe5, 0x22, 0xa7, 0x93, 0xb1, 0x76, 0x22, 0x44, 0x3e, 0x30, 0x88, 0xd1, 0x16, 0x18, 0xa3, 0xcd,
-	0x40, 0x3a, 0x5a, 0x26, 0xb2, 0xee, 0xa8, 0x13, 0xfc, 0x36, 0xa4, 0x58, 0x2d, 0x14, 0x33, 0x67,
-	0xdb, 0x72, 0x77, 0x44, 0x04, 0x1b, 0x77, 0x21, 0xcb, 0xce, 0x14, 0x23, 0x67, 0x24, 0x24, 0x64,
-	0x10, 0xd1, 0x4a, 0x0f, 0x3b, 0xb7, 0xea, 0x1e, 0xa7, 0xae, 0xca, 0x88, 0x40, 0x19, 0x0e, 0x83,
-	0x49, 0x19, 0x91, 0xc8, 0xfa, 0xef, 0xb3, 0x20, 0xf7, 0x1a, 0xf7, 0xdb, 0x98, 0xc0, 0x9f, 0x83,
-	0x5d, 0xd6, 0xd8, 0xe6, 0x10, 0x3b, 0x75, 0x9b, 0xf6, 0x92, 0xc6, 0x96, 0x72, 0x83, 0xa9, 0xe3,
-	0x1a, 0x78, 0x88, 0x1d, 0x23, 0xb2, 0x69, 0x4f, 0x47, 0x29, 0x38, 0xfc, 0x12, 0xe4, 0x4b, 0x5d,
-	0x1c, 0xd0, 0x92, 0xeb, 0x12, 0x9e, 0xd7, 0x7c, 0xf9, 0x68, 0x32, 0xd6, 0x3e, 0x4b, 0x5a, 0x87,
-	0xb9, 0x0c, 0xdb, 0x75, 0x89, 0x8e, 0xe6, 0x38, 0x96, 0xcf, 0x97, 0xb6, 0xe7, 0x47, 0xa1, 0x17,
-	0xd0, 0x57, 0xcd, 0x66, 0x9d, 0x93, 0x77, 0x39, 0x59, 0xca, 0x67, 0x67, 0x0a, 0x31, 0x7a, 0x94,
-	0x46, 0x89, 0xca, 0x32, 0x91, 0xe5, 0xb3, 0x6c, 0xc7, 0x98, 0xcd, 0x0f, 0xbc, 0x38, 0x6d, 0xda,
-	0x76, 0x8c, 0x93, 0x69, 0x93, 0x60, 0xe0, 0x57, 0x60, 0x87, 0x9d, 0xc0, 0x0a, 0xbb, 0xfc, 0xbc,
-	0x1d, 0x4e, 0x51, 0x27, 0x63, 0xed, 0x50, 0x3a, 0xaf, 0x1f, 0x76, 0x93, 0xe3, 0xca, 0x60, 0x58,
-	0x02, 0x05, 0xb6, 0x14, 0x0d, 0xdf, 0xb4, 0x1a, 0xea, 0x5f, 0x32, 0xfc, 0x33, 0x48, 0x5d, 0xc3,
-	0xe9, 0xc9, 0xa0, 0xa0, 0xac, 0x07, 0xd3, 0x0c, 0xf8, 0x0d, 0xd8, 0x9f, 0x1b, 0xea, 0x24, 0x1c,
-	0x8e, 0xd4, 0xbf, 0x0a, 0x91, 0xc7, 0x93, 0xb1, 0xa6, 0x2e, 0x8b, 0x44, 0x0c, 0xa3, 0xa3, 0x45,
-	0xd6, 0x34, 0x16, 0xd6, 0xd1, 0x42, 0xe6, 0x6f, 0xab, 0x63, 0xe1, 0xe3, 0x20, 0x11, 0x49, 0x33,
-	0x60, 0x1d, 0xc0, 0xb9, 0xaa, 0x19, 0xb8, 0x3c, 0xaf, 0xea, 0xf7, 0xa2, 0x04, 0xb4, 0xc9, 0x58,
-	0x7b, 0xb4, 0x1c, 0x0e, 0x4e, 0x60, 0x3a, 0x5a, 0xc1, 0x85, 0x3f, 0x11, 0xd7, 0x84, 0xfa, 0x67,
-	0x36, 0xf7, 0x77, 0x9e, 0xef, 0x5c, 0xf0, 0xdb, 0xe6, 0x82, 0xd9, 0xe4, 0xcb, 0x82, 0x09, 0xea,
-	0x88, 0x43, 0xf5, 0xff, 0x00, 0x90, 0x6b, 0x62, 0x3e, 0x50, 0x5e, 0x80, 0x82, 0xf8, 0x55, 0xc5,
-	0xf4, 0xdb, 0x90, 0xdc, 0x2e, 0x17, 0x23, 0xe5, 0x6e, 0x23, 0x10, 0x7e, 0x1d, 0xa5, 0xf1, 0xf0,
-	0xa7, 0x00, 0x08, 0x03, 0xaf, 0x28, 0x71, 0xf7, 0x3c, 0x98, 0x8c, 0x35, 0x98, 0x62, 0x8b, 0x4a,
-	0x92, 0x90, 0x6c, 0x6c, 0x5f, 0x62, 0xdf, 0x1e, 0x59, 0x36, 0xc5, 0x81, 0x33, 0x7a, 0x1d, 0xf3,
-	0x52, 0x2e, 0xc8, 0x63, 0xdb, 0x65, 0x7e, 0xc3, 0x17, 0x00, 0xa3, 0xcf, 0xc6, 0x76, 0x9a, 0x02,
-	0x7f, 0x09, 0x94, 0xb4, 0x05, 0xdd, 0xf1, 0xa2, 0x2e, 0xc8, 0x45, 0xbd, 0x28, 0x63, 0x90, 0x3b,
-	0x1d, 0x2d, 0xf1, 0xd8, 0x41, 0x50, 0x38, 0x08, 0x5c, 0xcb, 0xeb, 0x7b, 0x54, 0x3d, 0x2a, 0x66,
-	0xce, 0xb2, 0xf2, 0x41, 0x08, 0xf3, 0x19, 0x3e, 0x73, 0xea, 0x48, 0x42, 0xc2, 0x5f, 0x80, 0x82,
-	0x39, 0xf4, 0x68, 0x2d, 0x60, 0x6d, 0x32, 0x20, 0x58, 0x7d, 0xb0, 0x54, 0x13, 0x43, 0x8f, 0x1a,
-	0x61, 0x60, 0x74, 0x04, 0x80, 0xd5, 0x84, 0x4c, 0x80, 0xaf, 0x80, 0x52, 0x09, 0x83, 0x98, 0xdf,
-	0x48, 0xce, 0x48, 0xcc, 0x9a, 0xe3, 0xc5, 0xfa, 0x74, 0xe6, 0x88, 0xe9, 0x9c, 0x59, 0x62, 0xc1,
-	0x9f, 0x81, 0x1d, 0x33, 0xb0, 0xdb, 0x3e, 0xae, 0x47, 0x24, 0xec, 0xa8, 0x2a, 0x17, 0x39, 0x9e,
-	0x8c, 0xb5, 0x83, 0x24, 0x12, 0xee, 0x34, 0x22, 0xe6, 0x65, 0x7d, 0x36, 0xc7, 0xc2, 0xaf, 0xc1,
-	0x6e, 0x12, 0x4f, 0xc5, 0x8e, 0x71, 0xac, 0x6a, 0xfc, 0xd6, 0x93, 0x9a, 0x34, 0x89, 0xde, 0x70,
-	0x98, 0x5b, 0x47, 0x29, 0x34, 0xbb, 0xdb, 0x92, 0x75, 0xa3, 0x37, 0xe8, 0x74, 0x7c, 0xac, 0x16,
-	0x17, 0xb3, 0x30, 0xe5, 0xc7, 0x02, 0xa0, 0xa3, 0x05, 0x06, 0xbc, 0x96, 0x46, 0x54, 0x25, 0xec,
-	0xf7, 0xed, 0xc0, 0x8d, 0xd5, 0xcf, 0x17, 0x1f, 0x1a, 0xf3, 0x11, 0xe5, 0x24, 0x18, 0x79, 0x42,
-	0x4d, 0x79, 0xac, 0xbc, 0xd0, 0x20, 0x08, 0x30, 0x99, 0x4d, 0xd9, 0x1f, 0xf1, 0xd2, 0x94, 0xca,
-	0x8b, 0x70, 0xbf, 0x3c, 0x67, 0x17, 0x28, 0xec, 0xe5, 0x63, 0x0e, 0x29, 0x26, 0x81, 0xed, 0xcf,
-	0x64, 0xce, 0xb9, 0x8c, 0x14, 0x10, 0x4e, 0x10, 0xb2, 0xd0, 0x12, 0x8d, 0x7d, 0x99, 0x06, 0x25,
-	0x38, 0x8e, 0x9b, 0xa3, 0x08, 0xc7, 0x2a, 0xe6, 0xc7, 0x92, 0xbe, 0x4c, 0xcc, 0x9d, 0x06, 0x65,
-	0x5e, 0x1d, 0xc9, 0x58, 0x56, 0x60, 0x62, 0x79, 0x8d, 0x47, 0x0d, 0xef, 0x3b, 0xcc, 0xe7, 0x67,
-	0x56, 0x4e, 0x6d, 0x42, 0xbe, 0xc5, 0x23, 0x23, 0xf6, 0xbe, 0x63, 0x05, 0x96, 0x22, 0xb0, 0xa1,
-	0x93, 0x32, 0x58, 0x36, 0xe9, 0x62, 0xb5, 0xcb, 0x65, 0xa4, 0xeb, 0x6c, 0x41, 0xc6, 0xf0, 0x19,
-	0x4c, 0x47, 0x2b, 0xb8, 0xf0, 0x2d, 0x38, 0x9c, 0x5b, 0x07, 0x9d, 0x8e, 0x37, 0x44, 0x76, 0xd0,
-	0xc5, 0x6a, 0x8f, 0x6b, 0xea, 0x93, 0xb1, 0x76, 0xba, 0xac, 0xc9, 0x71, 0x06, 0x61, 0x40, 0x1d,
-	0xad, 0xe4, 0xc3, 0x5f, 0x83, 0xe3, 0x55, 0xf6, 0xe6, 0x30, 0x50, 0x3d, 0x2e, 0x2d, 0x3d, 0xc3,
-	0xee, 0x91, 0x36, 0xe8, 0x30, 0xd0, 0xd1, 0x7d, 0x32, 0xec, 0x32, 0x98, 0xb9, 0x9a, 0xc3, 0xa0,
-	0x16, 0xc5, 0xea, 0x6f, 0xb8, 0xb2, 0xf4, 0x49, 0x25, 0x65, 0x3a, 0x0c, 0x8c, 0x30, 0x8a, 0x75,
-	0xb4, 0xc8, 0x62, 0xd7, 0xb0, 0x30, 0xbd, 0xa9, 0x37, 0xd4, 0x5b, 0x2e, 0x21, 0x5d, 0xc3, 0x89,
-	0xc4, 0x07, 0x46, 0x9d, 0xe3, 0xf4, 0xdf, 0x65, 0xc0, 0x16, 0xc2, 0x1f, 0x06, 0x38, 0xa6, 0xf0,
-	0x02, 0xe4, 0x6b, 0x11, 0x26, 0x36, 0xf5, 0xc2, 0x80, 0x8f, 0xdd, 0xbd, 0xe7, 0x4a, 0x32, 0xbc,
-	0x67, 0x76, 0x34, 0x87, 0xc0, 0x2f, 0xa6, 0x0f, 0x08, 0x55, 0x4c, 0xfa, 0x42, 0x02, 0x16, 0x46,
-	0x34, 0x7d, 0x5d, 0x7c, 0x31, 0x9d, 0xed, 0xfc, 0x69, 0x3f, 0x87, 0x09, 0x23, 0x4a, 0x9c, 0xfa,
-	0xd7, 0x60, 0x1b, 0xe1, 0x38, 0x0a, 0x83, 0x18, 0x43, 0x15, 0x6c, 0x35, 0x06, 0x8e, 0x83, 0xe3,
-	0x98, 0xc7, 0xb1, 0x8d, 0xa6, 0x4b, 0xf8, 0x00, 0xe4, 0xd8, 0x23, 0x71, 0x10, 0x8b, 0xc9, 0x8e,
-	0x92, 0xd5, 0xf9, 0x3f, 0x32, 0x52, 0xf0, 0x70, 0x0f, 0x80, 0x6a, 0x48, 0x1b, 0xd4, 0x26, 0x14,
-	0xbb, 0xca, 0x1a, 0x3c, 0x04, 0x4a, 0xf2, 0x04, 0xe2, 0x36, 0x76, 0xe7, 0x28, 0x19, 0xb8, 0x0f,
-	0x76, 0x10, 0x8e, 0x67, 0x86, 0x75, 0xb8, 0x0b, 0xb6, 0xaf, 0x3d, 0xdf, 0xe7, 0xab, 0x0d, 0xe6,
-	0x66, 0x6d, 0x5c, 0x22, 0x4e, 0xcf, 0xbb, 0xc3, 0xca, 0x26, 0x53, 0xb9, 0xc4, 0x31, 0x25, 0xe1,
-	0x88, 0x21, 0xf8, 0x53, 0x46, 0xc9, 0xc2, 0x87, 0xe0, 0xa8, 0xec, 0xdb, 0xce, 0x6d, 0x2f, 0xf4,
-	0xf9, 0xd3, 0xba, 0x1e, 0x12, 0xda, 0x1c, 0xa2, 0xa1, 0xe2, 0xc2, 0x47, 0xe0, 0xf8, 0x26, 0x68,
-	0xaf, 0x74, 0x62, 0x78, 0x04, 0x3e, 0xe3, 0x23, 0x3f, 0x65, 0xee, 0xc0, 0x63, 0x70, 0x70, 0x13,
-	0xb8, 0x4b, 0x8e, 0xee, 0xf9, 0x1f, 0x37, 0x44, 0x3c, 0xc9, 0x88, 0x63, 0xfc, 0xeb, 0x2b, 0xcb,
-	0x6a, 0xd5, 0xaa, 0x66, 0xeb, 0x65, 0xcd, 0xb2, 0x6a, 0xef, 0x4c, 0xa4, 0xac, 0xb1, 0xa8, 0xb9,
-	0xd9, 0x32, 0x4b, 0x97, 0x26, 0x52, 0x32, 0xf0, 0x14, 0x9c, 0x2c, 0xe1, 0x5a, 0x2f, 0x6b, 0xa8,
-	0x65, 0xd5, 0xaa, 0xdf, 0x28, 0xeb, 0x50, 0x05, 0x87, 0x12, 0x61, 0xee, 0xd9, 0x98, 0x49, 0xbd,
-	0xb9, 0xa9, 0xa1, 0x9b, 0xd7, 0xca, 0x26, 0xcf, 0x0f, 0x33, 0x94, 0x2c, 0x4b, 0xc9, 0xc2, 0x73,
-	0xf0, 0xb4, 0x6c, 0x95, 0x2a, 0xd7, 0xaf, 0x6a, 0x96, 0xd9, 0xaa, 0x9b, 0x26, 0x6a, 0xd5, 0x6b,
-	0xa8, 0xd9, 0x6a, 0xbe, 0x6f, 0xa1, 0xf7, 0xe9, 0xa8, 0x72, 0xf0, 0x87, 0xa0, 0x78, 0x3f, 0x36,
-	0x09, 0x75, 0x0b, 0x7e, 0x0e, 0x9e, 0xdc, 0x8f, 0x62, 0x9b, 0x6e, 0xc3, 0xa7, 0x40, 0xbf, 0x34,
-	0xad, 0xd2, 0xaf, 0xfe, 0xf7, 0x86, 0x79, 0x58, 0x04, 0x8f, 0x57, 0xe3, 0x92, 0xcd, 0x00, 0x7c,
-	0x02, 0x1e, 0xae, 0x46, 0xb0, 0x8d, 0x76, 0x58, 0x09, 0xbd, 0x2c, 0x5d, 0x59, 0xf5, 0xda, 0x55,
-	0xb5, 0xd9, 0x50, 0x5c, 0xb8, 0x0b, 0xb6, 0xaa, 0xb5, 0x16, 0x33, 0x29, 0x1f, 0x33, 0xb0, 0x00,
-	0xb6, 0xcd, 0xf7, 0x4d, 0x13, 0x55, 0x4b, 0x96, 0xf2, 0xa7, 0xf5, 0xf3, 0xdf, 0x02, 0x30, 0x1f,
-	0x90, 0x30, 0x07, 0xd6, 0xaf, 0xdf, 0x2a, 0x6b, 0x30, 0x0f, 0xb2, 0x96, 0x59, 0x6a, 0x98, 0x0a,
-	0xc3, 0xe7, 0xab, 0xb5, 0x56, 0xa3, 0x89, 0xcc, 0x46, 0x43, 0x59, 0x87, 0x07, 0x60, 0xdf, 0xb4,
-	0xcc, 0x4a, 0xf3, 0xaa, 0x56, 0x6d, 0xa1, 0x9b, 0x6a, 0xd5, 0x44, 0xca, 0x06, 0x54, 0xc0, 0xee,
-	0xbb, 0x52, 0xb3, 0xf2, 0x6a, 0x6a, 0xd9, 0x64, 0x9f, 0xd8, 0xaa, 0x55, 0xae, 0x5b, 0xa8, 0x54,
-	0x31, 0xd1, 0xd4, 0x9c, 0x65, 0x40, 0xae, 0x3b, 0xb5, 0xe4, 0x9e, 0xbf, 0x00, 0xf9, 0x26, 0xb1,
-	0x83, 0x38, 0x0a, 0x09, 0x85, 0xcf, 0xe5, 0xc5, 0x5e, 0xd2, 0x6c, 0x49, 0x8f, 0x9f, 0xec, 0xcf,
-	0xd6, 0xa2, 0xd5, 0xf4, 0xb5, 0xb3, 0xcc, 0x8f, 0x33, 0xe5, 0xc3, 0x8f, 0xff, 0x3a, 0x5d, 0xfb,
-	0xf8, 0xe9, 0x34, 0xf3, 0xf7, 0x4f, 0xa7, 0x99, 0x7f, 0x7e, 0x3a, 0xcd, 0xfc, 0xe1, 0xdf, 0xa7,
-	0x6b, 0xed, 0x1c, 0xff, 0xe3, 0xff, 0xe5, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xc2, 0x76, 0x9b,
-	0x6a, 0x41, 0x10, 0x00, 0x00,
+	// 1870 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x58, 0x7f, 0x72, 0xdb, 0xb8,
+	0x15, 0xb6, 0x6c, 0xcb, 0xb6, 0xe0, 0x5f, 0x0c, 0x6c, 0xc7, 0x8c, 0x93, 0x98, 0x5e, 0x6e, 0x37,
+	0x75, 0x3d, 0x43, 0xa7, 0xcd, 0x76, 0x3a, 0xd3, 0x9d, 0xed, 0xa4, 0x92, 0xcc, 0xac, 0x5d, 0x33,
+	0x92, 0x02, 0xc9, 0x49, 0xfa, 0x97, 0x4a, 0x51, 0x90, 0xc4, 0x9a, 0x22, 0x19, 0x00, 0xf2, 0x4a,
+	0x7b, 0x81, 0x4e, 0x6f, 0xd0, 0x43, 0xb4, 0xf7, 0x48, 0xdb, 0x9d, 0x4e, 0x4f, 0xa0, 0xb6, 0xe9,
+	0x0d, 0x74, 0x81, 0x76, 0x00, 0x50, 0x12, 0x28, 0xc9, 0xed, 0x7f, 0xc2, 0x7b, 0xdf, 0xf7, 0xf1,
+	0xe1, 0x01, 0xef, 0x3d, 0xd8, 0x60, 0x97, 0xc4, 0x5e, 0xdc, 0x78, 0x4e, 0x62, 0xef, 0x3c, 0x26,
+	0x11, 0x8b, 0x60, 0x56, 0x18, 0x8e, 0xac, 0xb6, 0xcf, 0x3a, 0xbd, 0xc6, 0xb9, 0x17, 0x75, 0x9f,
+	0xb7, 0xa3, 0x76, 0xf4, 0x5c, 0x78, 0x1b, 0xbd, 0x96, 0x58, 0x89, 0x85, 0xf8, 0x25, 0x59, 0xe6,
+	0xf7, 0xeb, 0x60, 0xd5, 0x66, 0x5e, 0x13, 0x7e, 0x0e, 0x56, 0x4b, 0x6e, 0x17, 0xeb, 0x99, 0x93,
+	0xcc, 0x69, 0xae, 0xb0, 0x3b, 0x1a, 0x1a, 0x9b, 0x03, 0xb7, 0x1b, 0x7c, 0x65, 0x86, 0x6e, 0x17,
+	0x9b, 0x48, 0x38, 0xa1, 0x05, 0xd6, 0x2f, 0x5c, 0xe6, 0x5e, 0xf8, 0x44, 0x5f, 0x16, 0xb8, 0xbd,
+	0xd1, 0xd0, 0xd8, 0x95, 0xb8, 0xa6, 0xcb, 0x5c, 0xab, 0xe9, 0x13, 0x13, 0x8d, 0x31, 0xf0, 0x0c,
+	0xac, 0xbd, 0xcb, 0x3b, 0x1c, 0xbd, 0x22, 0xd0, 0x70, 0x34, 0x34, 0x76, 0x24, 0xfa, 0x5b, 0x37,
+	0x90, 0xe0, 0x04, 0x01, 0xaf, 0x80, 0xe6, 0xf8, 0x94, 0xe1, 0xb0, 0x18, 0xf8, 0x38, 0x64, 0x37,
+	0xc8, 0xa1, 0xfa, 0xea, 0xc9, 0xca, 0x69, 0xae, 0xf0, 0x74, 0x34, 0x34, 0x1e, 0x49, 0x56, 0x20,
+	0x10, 0x96, 0x27, 0x20, 0x56, 0x8f, 0x04, 0xd4, 0x44, 0x73, 0x34, 0x88, 0xc0, 0x5e, 0xbe, 0x79,
+	0x87, 0x09, 0xf3, 0x29, 0x56, 0xd4, 0xb2, 0x42, 0xed, 0x64, 0x34, 0x34, 0x9e, 0x48, 0x35, 0x77,
+	0x0c, 0x4a, 0x0b, 0x2e, 0x22, 0xc3, 0x22, 0xd8, 0x91, 0xdf, 0xa9, 0x60, 0x4c, 0x84, 0xdc, 0x9a,
+	0x90, 0x7b, 0x3c, 0x1a, 0x1a, 0x87, 0xa9, 0xe0, 0x62, 0x8c, 0x49, 0xa2, 0x34, 0x43, 0x81, 0x0d,
+	0xa0, 0x5f, 0x85, 0x3e, 0xf3, 0xdd, 0x60, 0xf2, 0x89, 0x89, 0xdc, 0xba, 0x90, 0x7b, 0x36, 0x1a,
+	0x1a, 0xa6, 0x94, 0xf3, 0x25, 0xd2, 0x9a, 0x46, 0xa9, 0x28, 0xdf, 0xab, 0x03, 0x0b, 0x60, 0x27,
+	0xf1, 0x15, 0x83, 0x1e, 0x65, 0x98, 0xe8, 0x1b, 0x22, 0xf7, 0x47, 0xa3, 0xa1, 0xf1, 0x30, 0xad,
+	0xec, 0x49, 0x80, 0x89, 0x66, 0x18, 0x3c, 0x81, 0x69, 0x4b, 0x95, 0xb9, 0x0c, 0xeb, 0x39, 0x21,
+	0xa4, 0x24, 0x70, 0x46, 0xc8, 0xa2, 0x1c, 0x66, 0xa2, 0x45, 0xe4, 0x79, 0xcd, 0x5a, 0x74, 0x8b,
+	0x43, 0x1d, 0xfc, 0x3f, 0x4d, 0xc6, 0x61, 0x73, 0x9a, 0x82, 0x0c, 0x5f, 0x82, 0xed, 0x6a, 0xe8,
+	0xc6, 0xb4, 0x13, 0xb1, 0x62, 0xd4, 0x0b, 0x99, 0xbe, 0x79, 0x92, 0x39, 0x5d, 0x29, 0x3c, 0x1a,
+	0x0d, 0x8d, 0x03, 0xa9, 0x46, 0x13, 0xb7, 0xe5, 0x71, 0xbf, 0x89, 0xd2, 0x78, 0xe8, 0x80, 0x07,
+	0x6f, 0x7a, 0x11, 0x73, 0x0b, 0xae, 0x77, 0x8b, 0xc3, 0x66, 0x61, 0xc0, 0x30, 0xd5, 0xb7, 0x84,
+	0xc8, 0xf1, 0x68, 0x68, 0x1c, 0x49, 0x91, 0x0f, 0x1c, 0x62, 0x35, 0x24, 0xc6, 0x6a, 0x70, 0x90,
+	0x89, 0xe6, 0x89, 0xbc, 0x3a, 0x2a, 0x04, 0xbf, 0x8d, 0x18, 0xd6, 0xb7, 0x4f, 0x32, 0xa7, 0x1b,
+	0x6a, 0x75, 0xc4, 0x04, 0x5b, 0x77, 0x11, 0xcf, 0xce, 0x18, 0xa3, 0x66, 0x24, 0x22, 0xa4, 0x17,
+	0xb3, 0x62, 0x07, 0x7b, 0xb7, 0xfa, 0x8e, 0xa0, 0x2e, 0xca, 0x88, 0x44, 0x59, 0x1e, 0x87, 0x29,
+	0x19, 0x51, 0xc8, 0xe6, 0xef, 0xb3, 0x60, 0xed, 0x35, 0xee, 0x36, 0x30, 0x81, 0xbf, 0x00, 0x5b,
+	0xbc, 0xb0, 0xed, 0x3e, 0xf6, 0x2a, 0x2e, 0xeb, 0x24, 0x85, 0xad, 0xe4, 0x06, 0x33, 0xaf, 0x69,
+	0xe1, 0x3e, 0xf6, 0xac, 0xd8, 0x65, 0x1d, 0x13, 0xa5, 0xe0, 0xf0, 0x4b, 0x90, 0xcb, 0xb7, 0x71,
+	0xc8, 0xf2, 0xcd, 0x26, 0x11, 0x79, 0xcd, 0x15, 0x0e, 0x46, 0x43, 0xe3, 0x41, 0x52, 0x3a, 0xdc,
+	0x65, 0xb9, 0xcd, 0x26, 0x31, 0xd1, 0x14, 0xc7, 0xf3, 0xf9, 0xca, 0xf5, 0x83, 0x38, 0xf2, 0x43,
+	0x76, 0x59, 0xab, 0x55, 0x04, 0x79, 0x4b, 0x90, 0x95, 0x7c, 0xb6, 0xc6, 0x10, 0xab, 0xc3, 0x58,
+	0x9c, 0xa8, 0xcc, 0x13, 0x79, 0x3e, 0x0b, 0x2e, 0xc5, 0xbc, 0x7f, 0xe0, 0xd9, 0x6e, 0xd3, 0x70,
+	0x29, 0x4e, 0xba, 0x4d, 0x82, 0x81, 0x5f, 0x81, 0x4d, 0xbe, 0x03, 0x27, 0x6a, 0x8b, 0xfd, 0xb6,
+	0x04, 0x45, 0x1f, 0x0d, 0x8d, 0x7d, 0x65, 0xbf, 0x41, 0xd4, 0x4e, 0xb6, 0xab, 0x82, 0x61, 0x1e,
+	0x6c, 0xf3, 0xa5, 0x2c, 0xf8, 0x9a, 0x53, 0xd5, 0xff, 0x9c, 0x11, 0xc7, 0xa0, 0x54, 0x8d, 0xa0,
+	0x27, 0x8d, 0x82, 0xf1, 0x1a, 0x4c, 0x33, 0xe0, 0x37, 0x60, 0x77, 0x6a, 0xa8, 0x90, 0xa8, 0x3f,
+	0xd0, 0xff, 0x22, 0x45, 0x9e, 0x8c, 0x86, 0x86, 0x3e, 0x2f, 0x12, 0x73, 0x8c, 0x89, 0x66, 0x59,
+	0xe3, 0x58, 0x78, 0x45, 0x4b, 0x99, 0xbf, 0x2e, 0x8e, 0x45, 0xb4, 0x83, 0x44, 0x24, 0xcd, 0x80,
+	0x15, 0x00, 0xa7, 0xaa, 0x76, 0xd8, 0x14, 0x79, 0xd5, 0xbf, 0x97, 0x57, 0xc0, 0x18, 0x0d, 0x8d,
+	0xc7, 0xf3, 0xe1, 0xe0, 0x04, 0x66, 0xa2, 0x05, 0x5c, 0xf8, 0x13, 0x39, 0x26, 0xf4, 0x3f, 0xf1,
+	0xbe, 0xbf, 0xf9, 0x62, 0xf3, 0x5c, 0x4c, 0x9b, 0x73, 0x6e, 0x53, 0x87, 0x05, 0x17, 0x34, 0x91,
+	0x80, 0x9a, 0xff, 0xd9, 0x04, 0x6b, 0x35, 0x2c, 0x1a, 0xca, 0x4b, 0xb0, 0x2d, 0x7f, 0x95, 0x30,
+	0xfb, 0x36, 0x22, 0xb7, 0xf3, 0x97, 0x91, 0x09, 0xb7, 0x15, 0x4a, 0xbf, 0x89, 0xd2, 0x78, 0xf8,
+	0x33, 0x00, 0xa4, 0x41, 0xdc, 0x28, 0x39, 0x7b, 0x1e, 0x8e, 0x86, 0x06, 0x4c, 0xb1, 0xe5, 0x4d,
+	0x52, 0x90, 0xbc, 0x6d, 0x5f, 0xe0, 0xc0, 0x1d, 0x38, 0x2e, 0xc3, 0xa1, 0x37, 0x78, 0x4d, 0xc5,
+	0x55, 0xde, 0x56, 0xdb, 0x76, 0x93, 0xfb, 0xad, 0x40, 0x02, 0xac, 0x2e, 0x6f, 0xdb, 0x69, 0x0a,
+	0xfc, 0x15, 0xd0, 0xd2, 0x16, 0x74, 0x27, 0x2e, 0xf5, 0xb6, 0x7a, 0xa9, 0x67, 0x65, 0x2c, 0x72,
+	0x67, 0xa2, 0x39, 0x1e, 0xdf, 0x08, 0x8a, 0x7a, 0x61, 0xd3, 0xf1, 0xbb, 0x3e, 0xd3, 0x0f, 0x4e,
+	0x32, 0xa7, 0x59, 0x75, 0x23, 0x84, 0xfb, 0xac, 0x80, 0x3b, 0x4d, 0xa4, 0x20, 0xe1, 0x2f, 0xc1,
+	0xb6, 0xdd, 0xf7, 0x59, 0x39, 0xe4, 0x65, 0xd2, 0x23, 0x58, 0x7f, 0x38, 0x77, 0x27, 0xfa, 0x3e,
+	0xb3, 0xa2, 0xd0, 0x6a, 0x49, 0x00, 0xbf, 0x13, 0x2a, 0x01, 0x5e, 0x02, 0xad, 0x18, 0x85, 0x54,
+	0x4c, 0x24, 0x6f, 0x20, 0x7b, 0xcd, 0xe1, 0xec, 0xfd, 0xf4, 0xa6, 0x88, 0x71, 0x9f, 0x99, 0x63,
+	0xc1, 0x9f, 0x83, 0x4d, 0x3b, 0x74, 0x1b, 0x01, 0xae, 0xc4, 0x24, 0x6a, 0xe9, 0xba, 0x10, 0x39,
+	0x1c, 0x0d, 0x8d, 0xbd, 0x24, 0x12, 0xe1, 0xb4, 0x62, 0xee, 0xe5, 0x75, 0x36, 0xc5, 0xc2, 0xaf,
+	0xc1, 0x56, 0x12, 0x4f, 0xd1, 0xa5, 0x98, 0xea, 0x86, 0x98, 0x7a, 0x4a, 0x91, 0x26, 0xd1, 0x5b,
+	0x1e, 0x77, 0x9b, 0x28, 0x85, 0xe6, 0xa7, 0x99, 0xac, 0x45, 0x5e, 0x5f, 0x53, 0xfd, 0x64, 0xf6,
+	0x34, 0xc7, 0x7c, 0x79, 0x1c, 0xe2, 0x34, 0xd3, 0x14, 0x3e, 0x20, 0x13, 0x4b, 0xb5, 0xd3, 0x6b,
+	0xb5, 0x02, 0xac, 0x7f, 0x36, 0x9b, 0xca, 0xb1, 0x08, 0x95, 0x80, 0xa9, 0x46, 0xc2, 0x80, 0xd7,
+	0x4a, 0x9f, 0x2b, 0x46, 0xdd, 0xae, 0x1b, 0x36, 0xa9, 0x6e, 0xce, 0xbe, 0x56, 0xa6, 0x7d, 0xce,
+	0x4b, 0x30, 0x6a, 0x9b, 0x1b, 0xf3, 0xf8, 0xae, 0x50, 0x2f, 0x0c, 0x31, 0x99, 0xb4, 0xea, 0x1f,
+	0x89, 0xfb, 0xad, 0xec, 0x8a, 0x08, 0xbf, 0xda, 0xac, 0x67, 0x28, 0xfc, 0xf9, 0x64, 0xf7, 0x19,
+	0x26, 0xa1, 0x1b, 0x4c, 0x64, 0xce, 0x84, 0x8c, 0x12, 0x10, 0x4e, 0x10, 0xaa, 0xd0, 0x1c, 0x8d,
+	0x1f, 0x6f, 0x95, 0x11, 0x4c, 0x69, 0x6d, 0x10, 0x63, 0xaa, 0x63, 0xb1, 0x2d, 0xe5, 0x78, 0xa9,
+	0x70, 0x5a, 0x8c, 0x7b, 0x4d, 0xa4, 0x62, 0xf9, 0x2d, 0x95, 0xcb, 0x6b, 0x3c, 0xa8, 0xfa, 0xdf,
+	0x61, 0xd1, 0x84, 0xb3, 0x6a, 0x6a, 0x13, 0xf2, 0x2d, 0x1e, 0x58, 0xd4, 0xff, 0x8e, 0xdf, 0xd2,
+	0x14, 0x81, 0x77, 0xae, 0x94, 0xc1, 0x71, 0x49, 0x1b, 0xeb, 0x6d, 0x21, 0xa3, 0xcc, 0xc4, 0x19,
+	0x19, 0x2b, 0xe0, 0x30, 0x13, 0x2d, 0xe0, 0xc2, 0xb7, 0x60, 0x7f, 0x6a, 0xed, 0xb5, 0x5a, 0x7e,
+	0x1f, 0xb9, 0x61, 0x1b, 0xeb, 0x1d, 0xa1, 0x69, 0x8e, 0x86, 0xc6, 0xf1, 0xbc, 0xa6, 0xc0, 0x59,
+	0x84, 0x03, 0x4d, 0xb4, 0x90, 0x0f, 0x7f, 0x03, 0x0e, 0x17, 0xd9, 0x6b, 0xfd, 0x50, 0xf7, 0x85,
+	0xb4, 0xf2, 0x96, 0xbb, 0x47, 0xda, 0x62, 0xfd, 0xd0, 0x44, 0xf7, 0xc9, 0xf0, 0x89, 0x32, 0x71,
+	0xd5, 0xfa, 0x61, 0x39, 0xa6, 0xfa, 0x6f, 0x85, 0xb2, 0x72, 0xa4, 0x8a, 0x32, 0xeb, 0x87, 0x56,
+	0x14, 0x53, 0x13, 0xcd, 0xb2, 0xa6, 0xc7, 0x22, 0x9b, 0x3a, 0x95, 0xd3, 0x2d, 0x9b, 0x7a, 0x28,
+	0x49, 0x1d, 0x39, 0x0b, 0xe8, 0xe4, 0x58, 0x12, 0x02, 0xfc, 0x29, 0xc8, 0x49, 0xc3, 0x9b, 0x4a,
+	0x55, 0x8e, 0xb5, 0xac, 0xfa, 0x1c, 0x48, 0xd8, 0x1f, 0xf8, 0xd7, 0xa7, 0x40, 0xf3, 0x77, 0x19,
+	0xb0, 0x8e, 0xf0, 0x87, 0x1e, 0xa6, 0x0c, 0x9e, 0x83, 0x5c, 0x39, 0xc6, 0xc4, 0x65, 0x7e, 0x14,
+	0x8a, 0xf6, 0xbf, 0xf3, 0x42, 0x4b, 0x86, 0xc8, 0xc4, 0x8e, 0xa6, 0x10, 0xf8, 0xc5, 0xf8, 0x21,
+	0xa3, 0xcb, 0x89, 0xb3, 0x9d, 0x80, 0xa5, 0x11, 0x8d, 0x5f, 0x39, 0x5f, 0x8c, 0x67, 0x8c, 0xf8,
+	0x13, 0x63, 0x0a, 0x93, 0x46, 0x94, 0x38, 0xcd, 0xaf, 0xc1, 0x06, 0xc2, 0x34, 0x8e, 0x42, 0x8a,
+	0xa1, 0x0e, 0xd6, 0xab, 0x3d, 0xcf, 0xc3, 0x94, 0x8a, 0x38, 0x36, 0xd0, 0x78, 0x09, 0x1f, 0x82,
+	0x35, 0xfe, 0x58, 0xed, 0x51, 0x39, 0x61, 0x50, 0xb2, 0x3a, 0xfb, 0x47, 0x46, 0x09, 0x1e, 0xee,
+	0x00, 0x50, 0x8a, 0x58, 0x95, 0xb9, 0x84, 0xe1, 0xa6, 0xb6, 0x04, 0xf7, 0x81, 0x96, 0x3c, 0xc5,
+	0x84, 0x8d, 0xcf, 0x3e, 0x2d, 0x03, 0x77, 0xc1, 0x26, 0xc2, 0x74, 0x62, 0x58, 0x86, 0x5b, 0x60,
+	0xe3, 0xda, 0x0f, 0x02, 0xb1, 0x5a, 0xe1, 0x6e, 0xde, 0x09, 0xf2, 0xc4, 0xeb, 0xf8, 0x77, 0x58,
+	0x5b, 0xe5, 0x2a, 0x17, 0x98, 0x32, 0x12, 0x0d, 0x38, 0x42, 0x3c, 0xa9, 0xb4, 0x2c, 0x7c, 0x04,
+	0x0e, 0x0a, 0x81, 0xeb, 0xdd, 0x76, 0xa2, 0x40, 0x3c, 0xf1, 0x2b, 0x11, 0x61, 0xb5, 0x3e, 0xea,
+	0x6b, 0x4d, 0xf8, 0x18, 0x1c, 0xde, 0x84, 0x8d, 0x85, 0x4e, 0x0c, 0x0f, 0xc0, 0x03, 0xd1, 0xef,
+	0x52, 0xe6, 0x16, 0x3c, 0x04, 0x7b, 0x37, 0x61, 0x73, 0xce, 0xd1, 0x3e, 0xfb, 0xdb, 0x8a, 0x8c,
+	0x27, 0x69, 0xb5, 0x9c, 0x7f, 0x7d, 0xe5, 0x38, 0xf5, 0x72, 0xc9, 0xae, 0xbf, 0x2a, 0x3b, 0x4e,
+	0xf9, 0x9d, 0x8d, 0xb4, 0x25, 0x1e, 0xb5, 0x30, 0x3b, 0x76, 0xfe, 0xc2, 0x46, 0x5a, 0x06, 0x1e,
+	0x83, 0xa3, 0x39, 0x5c, 0xfd, 0x55, 0x19, 0xd5, 0x9d, 0x72, 0xe9, 0x1b, 0x6d, 0x19, 0xea, 0x60,
+	0x5f, 0x21, 0x4c, 0x3d, 0x2b, 0x13, 0xa9, 0x37, 0x37, 0x65, 0x74, 0xf3, 0x5a, 0x5b, 0x15, 0xf9,
+	0xe1, 0x86, 0xbc, 0xe3, 0x68, 0x59, 0x78, 0x06, 0x9e, 0x15, 0x9c, 0x7c, 0xf1, 0xfa, 0xb2, 0xec,
+	0xd8, 0xf5, 0x8a, 0x6d, 0xa3, 0x7a, 0xa5, 0x8c, 0x6a, 0xf5, 0xda, 0xfb, 0x3a, 0x7a, 0x9f, 0x8e,
+	0x6a, 0x0d, 0xfe, 0x00, 0x9c, 0xdc, 0x8f, 0x4d, 0x42, 0x5d, 0x87, 0x9f, 0x81, 0xa7, 0xf7, 0xa3,
+	0xf8, 0x47, 0x37, 0xe0, 0x33, 0x60, 0x5e, 0xd8, 0x4e, 0xfe, 0xd7, 0xff, 0xfb, 0x83, 0x39, 0x78,
+	0x02, 0x9e, 0x2c, 0xc6, 0x25, 0x1f, 0x03, 0xf0, 0x29, 0x78, 0xb4, 0x18, 0xc1, 0x3f, 0xb4, 0xc9,
+	0xcf, 0xa1, 0x54, 0xae, 0xbf, 0xca, 0x5f, 0x39, 0xf5, 0x77, 0x57, 0xb5, 0xcb, 0x7a, 0xb5, 0x86,
+	0xec, 0x6a, 0x55, 0x6b, 0xc2, 0x1f, 0x82, 0xcf, 0x53, 0x8e, 0x52, 0x39, 0xf1, 0xc9, 0xd4, 0x5d,
+	0xbd, 0xb5, 0x4b, 0x1c, 0x88, 0xe1, 0x2e, 0x00, 0x1c, 0x55, 0x29, 0x5f, 0x95, 0x6a, 0x55, 0xed,
+	0x63, 0x06, 0x6e, 0x83, 0x0d, 0xfb, 0x7d, 0xcd, 0x46, 0xa5, 0xbc, 0xa3, 0xfd, 0x71, 0xf9, 0x2c,
+	0x02, 0x60, 0xda, 0x98, 0xe1, 0x1a, 0x58, 0xbe, 0x7e, 0xab, 0x2d, 0xc1, 0x1c, 0xc8, 0x3a, 0x76,
+	0xbe, 0x6a, 0x6b, 0x19, 0xb8, 0x07, 0x76, 0x6d, 0xc7, 0x2e, 0xd6, 0xae, 0xca, 0xa5, 0x3a, 0xba,
+	0x29, 0x95, 0x6c, 0xa4, 0x2d, 0x43, 0x0d, 0x6c, 0xbd, 0xcb, 0xd7, 0x8a, 0x97, 0x63, 0xcb, 0x0a,
+	0xbf, 0x08, 0x4e, 0xb9, 0x78, 0x5d, 0x47, 0xf9, 0xa2, 0x8d, 0xc6, 0xe6, 0x55, 0x0e, 0x14, 0x42,
+	0x63, 0x4b, 0xf6, 0xc5, 0x4b, 0x90, 0xab, 0x11, 0x37, 0xa4, 0x71, 0x44, 0x18, 0x7c, 0xa1, 0x2e,
+	0x76, 0x92, 0x92, 0x4c, 0x3a, 0xc1, 0xd1, 0xee, 0x64, 0x2d, 0x0b, 0xd2, 0x5c, 0x3a, 0xcd, 0xfc,
+	0x38, 0x53, 0xd8, 0xff, 0xf8, 0xaf, 0xe3, 0xa5, 0x8f, 0x9f, 0x8e, 0x33, 0x7f, 0xff, 0x74, 0x9c,
+	0xf9, 0xe7, 0xa7, 0xe3, 0xcc, 0x1f, 0xfe, 0x7d, 0xbc, 0xd4, 0x58, 0x13, 0xff, 0xa6, 0xf8, 0xf2,
+	0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x33, 0x62, 0x54, 0x24, 0xef, 0x10, 0x00, 0x00,
 }

tools/functional-tester/rpcpb/rpc.proto

@@ -100,23 +100,24 @@ enum FailureCase {
   DELAY_PEER_PORT_TX_RX_LEADER = 10;
   DELAY_PEER_PORT_TX_RX_ALL = 11;
 
-  FAILPOINTS = 100;
-
-  NO_FAIL = 200;
-  // TODO: support no-op of liveness duration
-  // NO_FAIL_LIVENESS = 201;
-
+  // NO_FAIL_WITH_STRESS runs no-op failure injection for specified period
+  // while stressers are still sending requests.
+  NO_FAIL_WITH_STRESS = 100;
+  // NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS runs no-op failure injection
+  // with all stressers stopped.
+  NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS = 101;
+
+  FAILPOINTS = 200;
   EXTERNAL = 300;
 }
 
 enum StressType {
   KV = 0;
   LEASE = 1;
-  NO_STRESS = 2;
-  ELECTION_RUNNER = 3;
-  WATCH_RUNNER = 4;
-  LOCK_RACER_RUNNER = 5;
-  LEASE_RUNNER = 6;
+  ELECTION_RUNNER = 2;
+  WATCH_RUNNER = 3;
+  LOCK_RACER_RUNNER = 4;
+  LEASE_RUNNER = 5;
 }
 
 message Tester {
@@ -140,12 +141,14 @@ message Tester {
 
   // FailureCases is the selected test cases to schedule.
   // If empty, run all failure cases.
-  // TODO: support no-op
   repeated string FailureCases = 31 [(gogoproto.moretags) = "yaml:\"failure-cases\""];
+  // FailureDelayMs is the delay duration after failure is injected.
+  // Useful when triggering snapshot or no-op failure cases.
+  uint32 FailureDelayMs = 32 [(gogoproto.moretags) = "yaml:\"failure-delay-ms\""];
   // FailureShuffle is true to randomize failure injecting order.
-  bool FailureShuffle = 32 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""];
+  bool FailureShuffle = 33 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""];
   // FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)).
-  repeated string FailpointCommands = 33 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""];
+  repeated string FailpointCommands = 34 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""];
 
   // RunnerExecPath is a path of etcd-runner binary.
   string RunnerExecPath = 41 [(gogoproto.moretags) = "yaml:\"runner-exec-path\""];
@@ -167,8 +170,12 @@ message Tester {
   int32 StressKeySuffixRangeTxn = 105 [(gogoproto.moretags) = "yaml:\"stress-key-suffix-range-txn\""];
   // StressKeyTxnOps is the number of operations per a transaction (max 64).
   int32 StressKeyTxnOps = 106 [(gogoproto.moretags) = "yaml:\"stress-key-txn-ops\""];
+
+  // StressClients is the number of concurrent stressing clients
+  // with "one" shared TCP connection.
+  int32 StressClients = 201 [(gogoproto.moretags) = "yaml:\"stress-clients\""];
   // StressQPS is the maximum number of stresser requests per second.
-  int32 StressQPS = 107 [(gogoproto.moretags) = "yaml:\"stress-qps\""];
+  int32 StressQPS = 202 [(gogoproto.moretags) = "yaml:\"stress-qps\""];
 }
 
 message Request {

tools/functional-tester/tester/cluster.go

@@ -276,27 +276,29 @@ func (clus *Cluster) updateFailures() {
 		case "KILL_ALL":
 			clus.failures = append(clus.failures, newFailureKillAll())
 		case "BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER":
-			clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower())
+			clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxOneFollower(clus))
 		case "BLACKHOLE_PEER_PORT_TX_RX_LEADER":
-			clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader())
+			clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxLeader(clus))
 		case "BLACKHOLE_PEER_PORT_TX_RX_ALL":
-			clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll())
+			clus.failures = append(clus.failures, newFailureBlackholePeerPortTxRxAll(clus))
 		case "DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER":
 			clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxOneFollower(clus))
 		case "DELAY_PEER_PORT_TX_RX_LEADER":
 			clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxLeader(clus))
 		case "DELAY_PEER_PORT_TX_RX_ALL":
 			clus.failures = append(clus.failures, newFailureDelayPeerPortTxRxAll(clus))
+		case "NO_FAIL_WITH_STRESS":
+			clus.failures = append(clus.failures, newFailureNoFailWithStress(clus))
+		case "NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS":
+			clus.failures = append(clus.failures, newFailureNoFailWithNoStressForLiveness(clus))
+		case "EXTERNAL":
+			clus.failures = append(clus.failures, newFailureExternal(clus.Tester.ExternalExecPath))
 		case "FAILPOINTS":
 			fpFailures, fperr := failpointFailures(clus)
 			if len(fpFailures) == 0 {
 				clus.lg.Info("no failpoints found!", zap.Error(fperr))
 			}
 			clus.failures = append(clus.failures, fpFailures...)
-		case "NO_FAIL":
-			clus.failures = append(clus.failures, newFailureNoOp())
-		case "EXTERNAL":
-			clus.failures = append(clus.failures, newFailureExternal(clus.Tester.ExternalExecPath))
 		}
 	}
 }
@@ -360,8 +362,8 @@ func (clus *Cluster) updateStresserChecker() {
 	)
 
 	cs := &compositeStresser{}
-	for idx := range clus.Members {
-		cs.stressers = append(cs.stressers, newStresser(clus, idx))
+	for _, m := range clus.Members {
+		cs.stressers = append(cs.stressers, newStresser(clus, m))
 	}
 	clus.stresser = cs
 
@@ -381,49 +383,6 @@ func (clus *Cluster) updateStresserChecker() {
 	)
 }
 
-func (clus *Cluster) startStresser() (err error) {
-	clus.lg.Info(
-		"starting stressers",
-		zap.Int("round", clus.rd),
-		zap.Int("case", clus.cs),
-	)
-	err = clus.stresser.Stress()
-	clus.lg.Info(
-		"started stressers",
-		zap.Int("round", clus.rd),
-		zap.Int("case", clus.cs),
-	)
-	return err
-}
-
-func (clus *Cluster) closeStresser() {
-	clus.lg.Info(
-		"closing stressers",
-		zap.Int("round", clus.rd),
-		zap.Int("case", clus.cs),
-	)
-	clus.stresser.Close()
-	clus.lg.Info(
-		"closed stressers",
-		zap.Int("round", clus.rd),
-		zap.Int("case", clus.cs),
-	)
-}
-
-func (clus *Cluster) pauseStresser() {
-	clus.lg.Info(
-		"pausing stressers",
-		zap.Int("round", clus.rd),
-		zap.Int("case", clus.cs),
-	)
-	clus.stresser.Pause()
-	clus.lg.Info(
-		"paused stressers",
-		zap.Int("round", clus.rd),
-		zap.Int("case", clus.cs),
-	)
-}
-
 func (clus *Cluster) checkConsistency() (err error) {
 	defer func() {
 		if err != nil {
@@ -436,7 +395,6 @@ func (clus *Cluster) checkConsistency() (err error) {
 			)
 			return
 		}
-		err = clus.startStresser()
 	}()
 
 	clus.lg.Info(
@@ -759,4 +717,12 @@ func (clus *Cluster) defrag() error {
 	return nil
 }
 
-func (clus *Cluster) Report() int64 { return clus.stresser.ModifiedKeys() }
+// GetFailureDelayDuration computes failure delay duration.
+func (clus *Cluster) GetFailureDelayDuration() time.Duration {
+	return time.Duration(clus.Tester.FailureDelayMs) * time.Millisecond
+}
+
+// Report reports the number of modified keys.
+func (clus *Cluster) Report() int64 {
+	return clus.stresser.ModifiedKeys()
+}

tools/functional-tester/tester/cluster_test.go

@@ -131,7 +131,10 @@ func Test_newCluster(t *testing.T) {
 				"DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER",
 				"DELAY_PEER_PORT_TX_RX_LEADER",
 				"DELAY_PEER_PORT_TX_RX_ALL",
+				"NO_FAIL_WITH_STRESS",
+				"NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS",
 			},
+			FailureDelayMs:          7000,
 			FailureShuffle:          true,
 			FailpointCommands:       []string{`panic("etcd-tester")`},
 			RunnerExecPath:          "/etcd-runner",
@@ -142,6 +145,7 @@ func Test_newCluster(t *testing.T) {
 			StressKeySuffixRange:    250000,
 			StressKeySuffixRangeTxn: 100,
 			StressKeyTxnOps:         10,
+			StressClients:           100,
 			StressQPS:               1000,
 		},
 	}

tools/functional-tester/tester/cluster_tester.go

@@ -19,6 +19,8 @@ import (
 	"os"
 	"time"
 
+	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
+
 	"go.uber.org/zap"
 )
 
@@ -29,7 +31,6 @@ const compactQPS = 50000
 // StartTester starts tester.
 func (clus *Cluster) StartTester() {
 	// TODO: upate status
-	clus.startStresser()
 
 	var preModifiedKey int64
 	for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ {
@@ -116,68 +117,80 @@ func (clus *Cluster) doRound() error {
 		zap.Int("round", clus.rd),
 		zap.Strings("failures", clus.failureStrings()),
 	)
-	for i, f := range clus.failures {
+
+	for i, fa := range clus.failures {
 		clus.cs = i
 
-		caseTotalCounter.WithLabelValues(f.Desc()).Inc()
+		caseTotalCounter.WithLabelValues(fa.Desc()).Inc()
+		clus.lg.Info(
+			"failure case START",
+			zap.Int("round", clus.rd),
+			zap.Int("case", clus.cs),
+			zap.String("desc", fa.Desc()),
+		)
 
 		clus.lg.Info("wait health before injecting failures")
 		if err := clus.WaitHealth(); err != nil {
 			return fmt.Errorf("wait full health error: %v", err)
 		}
 
+		stressStarted := false
+		if fa.FailureCase() != rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS {
+			clus.lg.Info(
+				"starting stressers before injecting failures",
+				zap.Int("round", clus.rd),
+				zap.Int("case", clus.cs),
+				zap.String("desc", fa.Desc()),
+			)
+			if err := clus.stresser.Stress(); err != nil {
+				return fmt.Errorf("start stresser error: %v", err)
+			}
+			stressStarted = true
+		}
+
 		clus.lg.Info(
-			"injecting failure",
+			"injecting",
 			zap.Int("round", clus.rd),
 			zap.Int("case", clus.cs),
-			zap.String("desc", f.Desc()),
+			zap.String("desc", fa.Desc()),
 		)
-		if err := f.Inject(clus); err != nil {
+		if err := fa.Inject(clus); err != nil {
 			return fmt.Errorf("injection error: %v", err)
 		}
-		clus.lg.Info(
-			"injected failure",
-			zap.Int("round", clus.rd),
-			zap.Int("case", clus.cs),
-			zap.String("desc", f.Desc()),
-		)
 
 		// if run local, recovering server may conflict
 		// with stressing client ports
 		// TODO: use unix for local tests
 		clus.lg.Info(
-			"recovering failure",
+			"recovering",
 			zap.Int("round", clus.rd),
 			zap.Int("case", clus.cs),
-			zap.String("desc", f.Desc()),
+			zap.String("desc", fa.Desc()),
 		)
-		if err := f.Recover(clus); err != nil {
+		if err := fa.Recover(clus); err != nil {
 			return fmt.Errorf("recovery error: %v", err)
 		}
-		clus.lg.Info(
-			"recovered failure",
-			zap.Int("round", clus.rd),
-			zap.Int("case", clus.cs),
-			zap.String("desc", f.Desc()),
-		)
 
-		clus.lg.Info("pausing stresser after failure recovery, before wait health")
-		clus.pauseStresser()
+		if stressStarted {
+			clus.lg.Info("pausing stresser after failure recovery, before wait health")
+			clus.stresser.Pause()
+		}
 
-		clus.lg.Info("wait health after recovering failures")
+		clus.lg.Info("wait health after recover")
 		if err := clus.WaitHealth(); err != nil {
 			return fmt.Errorf("wait full health error: %v", err)
 		}
-		clus.lg.Info("check consistency after recovering failures")
+
+		clus.lg.Info("check consistency after recover")
 		if err := clus.checkConsistency(); err != nil {
 			return fmt.Errorf("tt.checkConsistency error (%v)", err)
 		}
 
 		clus.lg.Info(
-			"failure case passed",
+			"failure case PASS",
 			zap.Int("round", clus.rd),
 			zap.Int("case", clus.cs),
-			zap.String("desc", f.Desc()),
+			zap.String("desc", fa.Desc()),
 		)
 	}
 
@@ -186,6 +199,7 @@ func (clus *Cluster) doRound() error {
 		zap.Int("round", clus.rd),
 		zap.Strings("failures", clus.failureStrings()),
 	)
+
 	return nil
 }
 
@@ -204,14 +218,6 @@ func (clus *Cluster) updateRevision() error {
 }
 
 func (clus *Cluster) compact(rev int64, timeout time.Duration) (err error) {
-	clus.lg.Info("pausing stresser before compact")
-	clus.pauseStresser()
-	defer func() {
-		if err == nil {
-			err = clus.startStresser()
-		}
-	}()
-
 	clus.lg.Info(
 		"compacting storage",
 		zap.Int64("current-revision", clus.currentRevision),
@@ -273,7 +279,13 @@ func (clus *Cluster) cleanup() error {
 	}
 	caseFailedTotalCounter.WithLabelValues(desc).Inc()
 
-	clus.closeStresser()
+	clus.lg.Info(
+		"closing stressers before archiving failure data",
+		zap.Int("round", clus.rd),
+		zap.Int("case", clus.cs),
+	)
+	clus.stresser.Close()
+
 	if err := clus.FailArchive(); err != nil {
 		clus.lg.Warn(
 			"cleanup failed",

tools/functional-tester/tester/failure.go

@@ -18,6 +18,8 @@ import (
 	"fmt"
 	"math/rand"
 	"time"
+
+	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
 )
 
 // Failure defines failure injection interface.
@@ -33,28 +35,32 @@ type Failure interface {
 	Recover(clus *Cluster) error
 	// Desc returns a description of the failure
 	Desc() string
+	// FailureCase returns "rpcpb.FailureCase" enum type.
+	FailureCase() rpcpb.FailureCase
 }
 
-type description string
-
-func (d description) Desc() string { return string(d) }
-
 type injectMemberFunc func(*Cluster, int) error
 type recoverMemberFunc func(*Cluster, int) error
 
 type failureByFunc struct {
-	description
+	desc
+	failureCase   rpcpb.FailureCase
 	injectMember  injectMemberFunc
 	recoverMember recoverMemberFunc
 }
 
-type failureFollower struct {
-	failureByFunc
-	last int
-	lead int
+func (f *failureByFunc) Desc() string {
+	if string(f.desc) != "" {
+		return string(f.desc)
+	}
+	return f.failureCase.String()
 }
 
-type failureLeader struct {
+func (f *failureByFunc) FailureCase() rpcpb.FailureCase {
+	return f.failureCase
+}
+
+type failureFollower struct {
 	failureByFunc
 	last int
 	lead int
@@ -82,22 +88,6 @@ func (f *failureFollower) updateIndex(clus *Cluster) error {
 	return nil
 }
 
-func (f *failureLeader) updateIndex(clus *Cluster) error {
-	idx, err := clus.GetLeader()
-	if err != nil {
-		return err
-	}
-	f.lead = idx
-	f.last = idx
-	return nil
-}
-
-type failureQuorum failureByFunc
-type failureAll failureByFunc
-
-// failureUntilSnapshot injects a failure and waits for a snapshot event
-type failureUntilSnapshot struct{ Failure }
-
 func (f *failureFollower) Inject(clus *Cluster) error {
 	if err := f.updateIndex(clus); err != nil {
 		return err
@@ -109,6 +99,33 @@ func (f *failureFollower) Recover(clus *Cluster) error {
 	return f.recoverMember(clus, f.last)
 }
 
+func (f *failureFollower) Desc() string {
+	if string(f.desc) != "" {
+		return string(f.desc)
+	}
+	return f.failureCase.String()
+}
+
+func (f *failureFollower) FailureCase() rpcpb.FailureCase {
+	return f.failureCase
+}
+
+type failureLeader struct {
+	failureByFunc
+	last int
+	lead int
+}
+
+func (f *failureLeader) updateIndex(clus *Cluster) error {
+	idx, err := clus.GetLeader()
+	if err != nil {
+		return err
+	}
+	f.lead = idx
+	f.last = idx
+	return nil
+}
+
 func (f *failureLeader) Inject(clus *Cluster) error {
 	if err := f.updateIndex(clus); err != nil {
 		return err
@@ -120,6 +137,12 @@ func (f *failureLeader) Recover(clus *Cluster) error {
 	return f.recoverMember(clus, f.last)
 }
 
+func (f *failureLeader) FailureCase() rpcpb.FailureCase {
+	return f.failureCase
+}
+
+type failureQuorum failureByFunc
+
 func (f *failureQuorum) Inject(clus *Cluster) error {
 	for i := range killMap(len(clus.Members), clus.rd) {
 		if err := f.injectMember(clus, i); err != nil {
@@ -138,6 +161,19 @@ func (f *failureQuorum) Recover(clus *Cluster) error {
 	return nil
 }
 
+func (f *failureQuorum) Desc() string {
+	if string(f.desc) != "" {
+		return string(f.desc)
+	}
+	return f.failureCase.String()
+}
+
+func (f *failureQuorum) FailureCase() rpcpb.FailureCase {
+	return f.failureCase
+}
+
+type failureAll failureByFunc
+
 func (f *failureAll) Inject(clus *Cluster) error {
 	for i := range clus.Members {
 		if err := f.injectMember(clus, i); err != nil {
@@ -156,6 +192,25 @@ func (f *failureAll) Recover(clus *Cluster) error {
 	return nil
 }
 
+func (f *failureAll) Desc() string {
+	if string(f.desc) != "" {
+		return string(f.desc)
+	}
+	return f.failureCase.String()
+}
+
+func (f *failureAll) FailureCase() rpcpb.FailureCase {
+	return f.failureCase
+}
+
+// failureUntilSnapshot injects a failure and waits for a snapshot event
+type failureUntilSnapshot struct {
+	desc        desc
+	failureCase rpcpb.FailureCase
+
+	Failure
+}
+
 const snapshotCount = 10000
 
 func (f *failureUntilSnapshot) Inject(clus *Cluster) error {
@@ -190,7 +245,14 @@ func (f *failureUntilSnapshot) Inject(clus *Cluster) error {
 }
 
 func (f *failureUntilSnapshot) Desc() string {
-	return f.Failure.Desc() + " for a long time and expect it to recover from an incoming snapshot"
+	if f.desc.Desc() != "" {
+		return f.desc.Desc()
+	}
+	return f.failureCase.String()
+}
+
+func (f *failureUntilSnapshot) FailureCase() rpcpb.FailureCase {
+	return f.failureCase
 }
 
 func killMap(size int, seed int) map[int]bool {
@@ -204,3 +266,7 @@ func killMap(size int, seed int) map[int]bool {
 		}
 	}
 }
+
+type desc string
+
+func (d desc) Desc() string { return string(d) }

tools/functional-tester/tester/failure_case_external.go

@@ -17,13 +17,17 @@ package tester
 import (
 	"fmt"
 	"os/exec"
+
+	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
 )
 
 type failureExternal struct {
 	Failure
 
-	description string
-	scriptPath  string
+	desc        string
+	failureCase rpcpb.FailureCase
+
+	scriptPath string
 }
 
 func (f *failureExternal) Inject(clus *Cluster) error {
@@ -34,11 +38,18 @@ func (f *failureExternal) Recover(clus *Cluster) error {
 	return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", clus.rd)).Run()
 }
 
-func (f *failureExternal) Desc() string { return f.description }
+func (f *failureExternal) Desc() string {
+	return f.desc
+}
+
+func (f *failureExternal) FailureCase() rpcpb.FailureCase {
+	return f.failureCase
+}
 
 func newFailureExternal(scriptPath string) Failure {
 	return &failureExternal{
-		description: fmt.Sprintf("external fault injector (script: %q)", scriptPath),
+		desc:        fmt.Sprintf("external fault injector (script: %q)", scriptPath),
+		failureCase: rpcpb.FailureCase_EXTERNAL,
 		scriptPath:  scriptPath,
 	}
 }

tools/functional-tester/tester/failure_case_failpoints.go

@@ -20,7 +20,8 @@ import (
 	"net/http"
 	"strings"
 	"sync"
-	"time"
+
+	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
 )
 
 type failpointStats struct {
@@ -42,14 +43,23 @@ func failpointFailures(clus *Cluster) (ret []Failure, err error) {
 		if len(fp) == 0 {
 			continue
 		}
+
 		fpFails := failuresFromFailpoint(fp, clus.Tester.FailpointCommands)
+
 		// wrap in delays so failpoint has time to trigger
 		for i, fpf := range fpFails {
 			if strings.Contains(fp, "Snap") {
 				// hack to trigger snapshot failpoints
-				fpFails[i] = &failureUntilSnapshot{fpf}
+				fpFails[i] = &failureUntilSnapshot{
+					desc:        desc(fpf.Desc()),
+					failureCase: rpcpb.FailureCase_FAILPOINTS,
+					Failure:     fpf,
+				}
 			} else {
-				fpFails[i] = &failureDelay{fpf, 3 * time.Second}
+				fpFails[i] = &failureDelay{
+					Failure:       fpf,
+					delayDuration: clus.GetFailureDelayDuration(),
+				}
 			}
 		}
 		ret = append(ret, fpFails...)
@@ -85,7 +95,8 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure)
 		fs = append(fs, []Failure{
 			&failureFollower{
 				failureByFunc: failureByFunc{
-					description:   description(fmt.Sprintf("failpoint %s (one: %s)", fp, fcmd)),
+					desc:          desc(fmt.Sprintf("failpoint %q (one: %q)", fp, fcmd)),
+					failureCase:   rpcpb.FailureCase_FAILPOINTS,
 					injectMember:  inject,
 					recoverMember: recov,
 				},
@@ -94,7 +105,8 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure)
 			},
 			&failureLeader{
 				failureByFunc: failureByFunc{
-					description:   description(fmt.Sprintf("failpoint %s (leader: %s)", fp, fcmd)),
+					desc:          desc(fmt.Sprintf("failpoint %q (leader: %q)", fp, fcmd)),
+					failureCase:   rpcpb.FailureCase_FAILPOINTS,
 					injectMember:  inject,
 					recoverMember: recov,
 				},
@@ -102,12 +114,14 @@ func failuresFromFailpoint(fp string, failpointCommands []string) (fs []Failure)
 				lead: -1,
 			},
 			&failureQuorum{
-				description:   description(fmt.Sprintf("failpoint %s (quorum: %s)", fp, fcmd)),
+				desc:          desc(fmt.Sprintf("failpoint %q (quorum: %q)", fp, fcmd)),
+				failureCase:   rpcpb.FailureCase_FAILPOINTS,
 				injectMember:  inject,
 				recoverMember: recov,
 			},
 			&failureAll{
-				description:   description(fmt.Sprintf("failpoint %s (all: %s)", fp, fcmd)),
+				desc:          desc(fmt.Sprintf("failpoint %q (all: %q)", fp, fcmd)),
+				failureCase:   rpcpb.FailureCase_FAILPOINTS,
 				injectMember:  inject,
 				recoverMember: recov,
 			},

tools/functional-tester/tester/failure_case_kill.go

@@ -26,7 +26,7 @@ func recoverKill(clus *Cluster, idx int) error {
 
 func newFailureKillOneFollower() Failure {
 	ff := failureByFunc{
-		description:   "kill one follower",
+		failureCase:   rpcpb.FailureCase_KILL_ONE_FOLLOWER,
 		injectMember:  injectKill,
 		recoverMember: recoverKill,
 	}
@@ -35,7 +35,7 @@ func newFailureKillOneFollower() Failure {
 
 func newFailureKillLeader() Failure {
 	ff := failureByFunc{
-		description:   "kill leader",
+		failureCase:   rpcpb.FailureCase_KILL_LEADER,
 		injectMember:  injectKill,
 		recoverMember: recoverKill,
 	}
@@ -44,7 +44,7 @@ func newFailureKillLeader() Failure {
 
 func newFailureKillQuorum() Failure {
 	return &failureQuorum{
-		description:   "kill quorum",
+		failureCase:   rpcpb.FailureCase_KILL_QUORUM,
 		injectMember:  injectKill,
 		recoverMember: recoverKill,
 	}
@@ -52,16 +52,22 @@ func newFailureKillQuorum() Failure {
 
 func newFailureKillAll() Failure {
 	return &failureAll{
-		description:   "kill all",
+		failureCase:   rpcpb.FailureCase_KILL_ALL,
 		injectMember:  injectKill,
 		recoverMember: recoverKill,
 	}
 }
 
 func newFailureKillOneFollowerForLongTime() Failure {
-	return &failureUntilSnapshot{newFailureKillOneFollower()}
+	return &failureUntilSnapshot{
+		failureCase: rpcpb.FailureCase_KILL_ONE_FOLLOWER_FOR_LONG,
+		Failure:     newFailureKillOneFollower(),
+	}
 }
 
 func newFailureKillLeaderForLongTime() Failure {
-	return &failureUntilSnapshot{newFailureKillLeader()}
+	return &failureUntilSnapshot{
+		failureCase: rpcpb.FailureCase_KILL_LEADER_FOR_LONG,
+		Failure:     newFailureKillLeader(),
+	}
 }

tools/functional-tester/tester/failure_case_network_blackhole.go

@@ -14,7 +14,9 @@
 
 package tester
 
-import "github.com/coreos/etcd/tools/functional-tester/rpcpb"
+import (
+	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
+)
 
 func injectBlackholePeerPortTxRx(clus *Cluster, idx int) error {
 	return clus.sendOperation(idx, rpcpb.Operation_BlackholePeerPortTxRx)
@@ -24,40 +26,40 @@ func recoverBlackholePeerPortTxRx(clus *Cluster, idx int) error {
 	return clus.sendOperation(idx, rpcpb.Operation_UnblackholePeerPortTxRx)
 }
 
-func newFailureBlackholePeerPortTxRxOneFollower() Failure {
+func newFailureBlackholePeerPortTxRxOneFollower(clus *Cluster) Failure {
 	ff := failureByFunc{
-		description:   "blackhole peer port on one follower",
+		failureCase:   rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ONE_FOLLOWER,
 		injectMember:  injectBlackholePeerPortTxRx,
 		recoverMember: recoverBlackholePeerPortTxRx,
 	}
 	f := &failureFollower{ff, -1, -1}
 	return &failureDelay{
 		Failure:       f,
-		delayDuration: triggerElectionDur,
+		delayDuration: clus.GetFailureDelayDuration(),
 	}
 }
 
-func newFailureBlackholePeerPortTxRxLeader() Failure {
+func newFailureBlackholePeerPortTxRxLeader(clus *Cluster) Failure {
 	ff := failureByFunc{
-		description:   "blackhole peer port on leader",
+		failureCase:   rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_LEADER,
 		injectMember:  injectBlackholePeerPortTxRx,
 		recoverMember: recoverBlackholePeerPortTxRx,
 	}
 	f := &failureLeader{ff, -1, -1}
 	return &failureDelay{
 		Failure:       f,
-		delayDuration: triggerElectionDur,
+		delayDuration: clus.GetFailureDelayDuration(),
 	}
 }
 
-func newFailureBlackholePeerPortTxRxAll() Failure {
+func newFailureBlackholePeerPortTxRxAll(clus *Cluster) Failure {
 	f := &failureAll{
-		description:   "blackhole peer port on all",
+		failureCase:   rpcpb.FailureCase_BLACKHOLE_PEER_PORT_TX_RX_ALL,
 		injectMember:  injectBlackholePeerPortTxRx,
 		recoverMember: recoverBlackholePeerPortTxRx,
 	}
 	return &failureDelay{
 		Failure:       f,
-		delayDuration: triggerElectionDur,
+		delayDuration: clus.GetFailureDelayDuration(),
 	}
 }

tools/functional-tester/tester/failure_case_network_slow.go

@@ -15,16 +15,12 @@
 package tester
 
 import (
-	"fmt"
 	"time"
 
 	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
 )
 
 const (
-	// delay duration to trigger leader election (default election timeout 1s)
-	triggerElectionDur = 5 * time.Second
-
 	// Wait more when it recovers from slow network, because network layer
 	// needs extra time to propagate traffic control (tc command) change.
 	// Otherwise, we get different hash values from the previous revision.
@@ -43,42 +39,39 @@ func recoverDelayPeerPortTxRx(clus *Cluster, idx int) error {
 }
 
 func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster) Failure {
-	desc := fmt.Sprintf("delay follower peer port by %d ms", clus.Tester.DelayLatencyMs)
 	ff := failureByFunc{
-		description:   description(desc),
+		failureCase:   rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER,
 		injectMember:  injectDelayPeerPortTxRx,
 		recoverMember: recoverDelayPeerPortTxRx,
 	}
 	f := &failureFollower{ff, -1, -1}
 	return &failureDelay{
 		Failure:       f,
-		delayDuration: triggerElectionDur,
+		delayDuration: clus.GetFailureDelayDuration(),
 	}
 }
 
 func newFailureDelayPeerPortTxRxLeader(clus *Cluster) Failure {
-	desc := fmt.Sprintf("delay leader peer port by %d ms", clus.Tester.DelayLatencyMs)
 	ff := failureByFunc{
-		description:   description(desc),
+		failureCase:   rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_LEADER,
 		injectMember:  injectDelayPeerPortTxRx,
 		recoverMember: recoverDelayPeerPortTxRx,
 	}
 	f := &failureLeader{ff, -1, -1}
 	return &failureDelay{
 		Failure:       f,
-		delayDuration: triggerElectionDur,
+		delayDuration: clus.GetFailureDelayDuration(),
 	}
 }
 
 func newFailureDelayPeerPortTxRxAll(clus *Cluster) Failure {
-	desc := fmt.Sprintf("delay all peer port by %d ms", clus.Tester.DelayLatencyMs)
 	f := &failureAll{
-		description:   description(desc),
+		failureCase:   rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ALL,
 		injectMember:  injectDelayPeerPortTxRx,
 		recoverMember: recoverDelayPeerPortTxRx,
 	}
 	return &failureDelay{
 		Failure:       f,
-		delayDuration: triggerElectionDur,
+		delayDuration: clus.GetFailureDelayDuration(),
 	}
 }

tools/functional-tester/tester/failure_case_no_fail.go

@@ -0,0 +1,99 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tester
+
+import (
+	"time"
+
+	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
+
+	"go.uber.org/zap"
+)
+
+type failureNoFailWithStress failureByFunc
+
+func (f *failureNoFailWithStress) Inject(clus *Cluster) error {
+	return nil
+}
+
+func (f *failureNoFailWithStress) Recover(clus *Cluster) error {
+	return nil
+}
+
+func (f *failureNoFailWithStress) Desc() string {
+	if f.desc.Desc() != "" {
+		return f.desc.Desc()
+	}
+	return f.failureCase.String()
+}
+
+func (f *failureNoFailWithStress) FailureCase() rpcpb.FailureCase {
+	return f.failureCase
+}
+
+func newFailureNoFailWithStress(clus *Cluster) Failure {
+	f := &failureNoFailWithStress{
+		failureCase: rpcpb.FailureCase_NO_FAIL_WITH_STRESS,
+	}
+	return &failureDelay{
+		Failure:       f,
+		delayDuration: clus.GetFailureDelayDuration(),
+	}
+}
+
+type failureNoFailWithNoStressForLiveness failureByFunc
+
+func (f *failureNoFailWithNoStressForLiveness) Inject(clus *Cluster) error {
+	clus.lg.Info(
+		"extra delay for liveness mode with no stresser",
+		zap.Int("round", clus.rd),
+		zap.Int("case", clus.cs),
+		zap.String("desc", f.Desc()),
+	)
+	time.Sleep(clus.GetFailureDelayDuration())
+
+	clus.lg.Info(
+		"wait health in liveness mode",
+		zap.Int("round", clus.rd),
+		zap.Int("case", clus.cs),
+		zap.String("desc", f.Desc()),
+	)
+	return clus.WaitHealth()
+}
+
+func (f *failureNoFailWithNoStressForLiveness) Recover(clus *Cluster) error {
+	return nil
+}
+
+func (f *failureNoFailWithNoStressForLiveness) Desc() string {
+	if f.desc.Desc() != "" {
+		return f.desc.Desc()
+	}
+	return f.failureCase.String()
+}
+
+func (f *failureNoFailWithNoStressForLiveness) FailureCase() rpcpb.FailureCase {
+	return f.failureCase
+}
+
+func newFailureNoFailWithNoStressForLiveness(clus *Cluster) Failure {
+	f := &failureNoFailWithNoStressForLiveness{
+		failureCase: rpcpb.FailureCase_NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS,
+	}
+	return &failureDelay{
+		Failure:       f,
+		delayDuration: clus.GetFailureDelayDuration(),
+	}
+}

tools/functional-tester/tester/failure_case_no_op.go

@@ -1,26 +0,0 @@
-// Copyright 2018 The etcd Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tester
-
-type failureNoOp failureByFunc
-
-func (f *failureNoOp) Inject(clus *Cluster) error  { return nil }
-func (f *failureNoOp) Recover(clus *Cluster) error { return nil }
-
-func newFailureNoOp() Failure {
-	return &failureNoOp{
-		description: "no failure",
-	}
-}

tools/functional-tester/tester/local-test.yaml

@@ -97,7 +97,10 @@ tester-config:
   - DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER
   - DELAY_PEER_PORT_TX_RX_LEADER
   - DELAY_PEER_PORT_TX_RX_ALL
+  - NO_FAIL_WITH_STRESS
+  - NO_FAIL_WITH_NO_STRESS_FOR_LIVENESS
 
+  failure-delay-ms: 7000
   failure-shuffle: true
   failpoint-commands:
   - panic("etcd-tester")
@@ -110,7 +113,6 @@ tester-config:
   stress-types:
   - KV
   - LEASE
-  # - NO_STRESS
   # - ELECTION_RUNNER
   # - WATCH_RUNNER
   # - LOCK_RACER_RUNNER
@@ -121,4 +123,6 @@ tester-config:
   stress-key-suffix-range: 250000
   stress-key-suffix-range-txn: 100
   stress-key-txn-ops: 10
+
+  stress-clients: 100
   stress-qps: 1000

tools/functional-tester/tester/stress.go

@@ -18,6 +18,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
+
 	"go.uber.org/zap"
 )
 
@@ -36,34 +38,31 @@ type Stresser interface {
 }
 
 // newStresser creates stresser from a comma separated list of stresser types.
-func newStresser(clus *Cluster, idx int) Stresser {
+func newStresser(clus *Cluster, m *rpcpb.Member) Stresser {
 	stressers := make([]Stresser, len(clus.Tester.StressTypes))
 	for i, stype := range clus.Tester.StressTypes {
 		clus.lg.Info("creating stresser", zap.String("type", stype))
 
 		switch stype {
-		case "NO_STRESS":
-			stressers[i] = &nopStresser{start: time.Now(), qps: int(clus.rateLimiter.Limit())}
-
 		case "KV":
 			// TODO: Too intensive stressing clients can panic etcd member with
 			// 'out of memory' error. Put rate limits in server side.
 			stressers[i] = &keyStresser{
 				lg:                clus.lg,
-				m:                 clus.Members[idx],
+				m:                 m,
 				keySize:           int(clus.Tester.StressKeySize),
 				keyLargeSize:      int(clus.Tester.StressKeySizeLarge),
 				keySuffixRange:    int(clus.Tester.StressKeySuffixRange),
 				keyTxnSuffixRange: int(clus.Tester.StressKeySuffixRangeTxn),
 				keyTxnOps:         int(clus.Tester.StressKeyTxnOps),
-				N:                 100,
+				clientsN:          int(clus.Tester.StressClients),
 				rateLimiter:       clus.rateLimiter,
 			}
 
 		case "LEASE":
 			stressers[i] = &leaseStresser{
 				lg:           clus.lg,
-				m:            clus.Members[idx],
+				m:            m,
 				numLeases:    10, // TODO: configurable
 				keysPerLease: 10, // TODO: configurable
 				rateLimiter:  clus.rateLimiter,
@@ -75,7 +74,7 @@ func newStresser(clus *Cluster, idx int) Stresser {
 				"election",
 				fmt.Sprintf("%v", time.Now().UnixNano()), // election name as current nano time
 				"--dial-timeout=10s",
-				"--endpoints", clus.Members[idx].EtcdClientEndpoint,
+				"--endpoints", m.EtcdClientEndpoint,
 				"--total-client-connections=10",
 				"--rounds=0", // runs forever
 				"--req-rate", fmt.Sprintf("%v", reqRate),
@@ -95,7 +94,7 @@ func newStresser(clus *Cluster, idx int) Stresser {
 				"--total-keys=1",
 				"--total-prefixes=1",
 				"--watch-per-prefix=1",
-				"--endpoints", clus.Members[idx].EtcdClientEndpoint,
+				"--endpoints", m.EtcdClientEndpoint,
 				"--rounds=0", // runs forever
 				"--req-rate", fmt.Sprintf("%v", reqRate),
 			}
@@ -106,7 +105,7 @@ func newStresser(clus *Cluster, idx int) Stresser {
 			args := []string{
 				"lock-racer",
 				fmt.Sprintf("%v", time.Now().UnixNano()), // locker name as current nano time
-				"--endpoints", clus.Members[idx].EtcdClientEndpoint,
+				"--endpoints", m.EtcdClientEndpoint,
 				"--total-client-connections=10",
 				"--rounds=0", // runs forever
 				"--req-rate", fmt.Sprintf("%v", reqRate),
@@ -117,7 +116,7 @@ func newStresser(clus *Cluster, idx int) Stresser {
 			args := []string{
 				"lease-renewer",
 				"--ttl=30",
-				"--endpoints", clus.Members[idx].EtcdClientEndpoint,
+				"--endpoints", m.EtcdClientEndpoint,
 			}
 			stressers[i] = newRunnerStresser(clus.Tester.RunnerExecPath, args, clus.rateLimiter, 0)
 		}

tools/functional-tester/tester/stress_key.go

@@ -44,14 +44,15 @@ type keyStresser struct {
 	keyTxnSuffixRange int
 	keyTxnOps         int
 
-	N int
-
 	rateLimiter *rate.Limiter
 
-	wg sync.WaitGroup
+	wg       sync.WaitGroup
+	clientsN int
 
+	ctx    context.Context
 	cancel func()
 	cli    *clientv3.Client
+
 	// atomicModifiedKeys records the number of keys created and deleted by the stresser.
 	atomicModifiedKeys int64
 
@@ -59,40 +60,37 @@ type keyStresser struct {
 }
 
 func (s *keyStresser) Stress() error {
-	// TODO: add backoff option
-	cli, err := s.m.CreateEtcdClient()
+	var err error
+	s.cli, err = s.m.CreateEtcdClient(grpc.WithBackoffMaxDelay(1 * time.Second))
 	if err != nil {
 		return fmt.Errorf("%v (%q)", err, s.m.EtcdClientEndpoint)
 	}
-	ctx, cancel := context.WithCancel(context.Background())
-
-	s.wg.Add(s.N)
-	s.cli = cli
-	s.cancel = cancel
+	s.ctx, s.cancel = context.WithCancel(context.Background())
 
+	s.wg.Add(s.clientsN)
 	var stressEntries = []stressEntry{
-		{weight: 0.7, f: newStressPut(cli, s.keySuffixRange, s.keySize)},
+		{weight: 0.7, f: newStressPut(s.cli, s.keySuffixRange, s.keySize)},
 		{
 			weight: 0.7 * float32(s.keySize) / float32(s.keyLargeSize),
-			f:      newStressPut(cli, s.keySuffixRange, s.keyLargeSize),
+			f:      newStressPut(s.cli, s.keySuffixRange, s.keyLargeSize),
 		},
-		{weight: 0.07, f: newStressRange(cli, s.keySuffixRange)},
-		{weight: 0.07, f: newStressRangeInterval(cli, s.keySuffixRange)},
-		{weight: 0.07, f: newStressDelete(cli, s.keySuffixRange)},
-		{weight: 0.07, f: newStressDeleteInterval(cli, s.keySuffixRange)},
+		{weight: 0.07, f: newStressRange(s.cli, s.keySuffixRange)},
+		{weight: 0.07, f: newStressRangeInterval(s.cli, s.keySuffixRange)},
+		{weight: 0.07, f: newStressDelete(s.cli, s.keySuffixRange)},
+		{weight: 0.07, f: newStressDeleteInterval(s.cli, s.keySuffixRange)},
 	}
 	if s.keyTxnSuffixRange > 0 {
 		// adjust to make up ±70% of workloads with writes
 		stressEntries[0].weight = 0.35
 		stressEntries = append(stressEntries, stressEntry{
 			weight: 0.35,
-			f:      newStressTxn(cli, s.keyTxnSuffixRange, s.keyTxnOps),
+			f:      newStressTxn(s.cli, s.keyTxnSuffixRange, s.keyTxnOps),
 		})
 	}
 	s.stressTable = createStressTable(stressEntries)
 
-	for i := 0; i < s.N; i++ {
-		go s.run(ctx)
+	for i := 0; i < s.clientsN; i++ {
+		go s.run()
 	}
 
 	s.lg.Info(
@@ -102,18 +100,18 @@ func (s *keyStresser) Stress() error {
 	return nil
 }
 
-func (s *keyStresser) run(ctx context.Context) {
+func (s *keyStresser) run() {
 	defer s.wg.Done()
 
 	for {
-		if err := s.rateLimiter.Wait(ctx); err == context.Canceled {
+		if err := s.rateLimiter.Wait(s.ctx); err == context.Canceled {
 			return
 		}
 
 		// TODO: 10-second is enough timeout to cover leader failure
 		// and immediate leader election. Find out what other cases this
 		// could be timed out.
-		sctx, scancel := context.WithTimeout(ctx, 10*time.Second)
+		sctx, scancel := context.WithTimeout(s.ctx, 10*time.Second)
 		err, modifiedKeys := s.stressTable.choose()(sctx)
 		scancel()
 		if err == nil {

tools/functional-tester/tester/stress_lease.go

@@ -33,8 +33,8 @@ import (
 
 const (
 	// time to live for lease
-	TTL      = 120
-	TTLShort = 2
+	defaultTTL      = 120
+	defaultTTLShort = 2
 )
 
 type leaseStresser struct {
@@ -201,7 +201,7 @@ func (ls *leaseStresser) createAliveLeases() {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			leaseID, err := ls.createLeaseWithKeys(TTL)
+			leaseID, err := ls.createLeaseWithKeys(defaultTTL)
 			if err != nil {
 				ls.lg.Debug(
 					"createLeaseWithKeys failed",
@@ -228,7 +228,7 @@ func (ls *leaseStresser) createShortLivedLeases() {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			leaseID, err := ls.createLeaseWithKeys(TTLShort)
+			leaseID, err := ls.createLeaseWithKeys(defaultTTLShort)
 			if err != nil {
 				return
 			}
@@ -323,7 +323,7 @@ func (ls *leaseStresser) keepLeaseAlive(leaseID int64) {
 			// if it is renewed, this means that invariant checking have at least ttl/2 time before lease exipres which is long enough for the checking to finish.
 			// if it is not renewed, we remove the lease from the alive map so that the lease doesn't exipre during invariant checking
 			renewTime, ok := ls.aliveLeases.read(leaseID)
-			if ok && renewTime.Add(TTL/2*time.Second).Before(time.Now()) {
+			if ok && renewTime.Add(defaultTTL/2*time.Second).Before(time.Now()) {
 				ls.aliveLeases.remove(leaseID)
 				ls.lg.Debug(
 					"keepLeaseAlive lease has not been renewed, dropped it",

tools/functional-tester/tester/stress_nop.go

@@ -1,31 +0,0 @@
-// Copyright 2018 The etcd Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package tester
-
-import "time"
-
-// nopStresser implements Stresser that does nothing
-type nopStresser struct {
-	start time.Time
-	qps   int
-}
-
-func (s *nopStresser) Stress() error { return nil }
-func (s *nopStresser) Pause()        {}
-func (s *nopStresser) Close()        {}
-func (s *nopStresser) ModifiedKeys() int64 {
-	return 0
-}
-func (s *nopStresser) Checker() Checker { return nil }