Merge pull request #9515 from gyuho/ftest

functional-tester: randomize failure injection sequence (by default)

tools/functional-tester/agent/handler.go

@@ -90,12 +90,12 @@ func (srv *Server) handleInitialStartEtcd(req *rpcpb.Request) (*rpcpb.Response,
 	}
 	srv.creatEtcdCmd()
 
-	srv.logger.Info("starting etcd process")
+	srv.logger.Info("starting etcd")
 	err = srv.startEtcdCmd()
 	if err != nil {
 		return nil, err
 	}
-	srv.logger.Info("started etcd process", zap.String("command-path", srv.etcdCmd.Path))
+	srv.logger.Info("started etcd", zap.String("command-path", srv.etcdCmd.Path))
 
 	// wait some time for etcd listener start
 	// before setting up proxy
@@ -248,15 +248,17 @@ func (srv *Server) startEtcdCmd() error {
 func (srv *Server) handleRestartEtcd() (*rpcpb.Response, error) {
 	srv.creatEtcdCmd()
 
-	srv.logger.Info("restarting etcd process")
+	srv.logger.Info("restarting etcd")
 	err := srv.startEtcdCmd()
 	if err != nil {
 		return nil, err
 	}
-	srv.logger.Info("restarted etcd process", zap.String("command-path", srv.etcdCmd.Path))
+	srv.logger.Info("restarted etcd", zap.String("command-path", srv.etcdCmd.Path))
 
 	// wait some time for etcd listener start
 	// before setting up proxy
+	// TODO: local tests should handle port conflicts
+	// with clients on restart
 	time.Sleep(time.Second)
 	if err = srv.startProxy(); err != nil {
 		return nil, err
@@ -269,21 +271,14 @@ func (srv *Server) handleRestartEtcd() (*rpcpb.Response, error) {
 }
 
 func (srv *Server) handleKillEtcd() (*rpcpb.Response, error) {
-	if srv.last != rpcpb.Operation_InitialStartEtcd && srv.last != rpcpb.Operation_RestartEtcd {
-		return &rpcpb.Response{
-			Success: false,
-			Status:  fmt.Sprintf("%q is not valid; last server operation was %q", rpcpb.Operation_KillEtcd.String(), srv.last.String()),
-		}, nil
-	}
-
 	srv.stopProxy()
 
-	srv.logger.Info("killing etcd process", zap.String("signal", syscall.SIGTERM.String()))
+	srv.logger.Info("killing etcd", zap.String("signal", syscall.SIGTERM.String()))
 	err := stopWithSig(srv.etcdCmd, syscall.SIGTERM)
 	if err != nil {
 		return nil, err
 	}
-	srv.logger.Info("killed etcd process", zap.String("signal", syscall.SIGTERM.String()))
+	srv.logger.Info("killed etcd", zap.String("signal", syscall.SIGTERM.String()))
 
 	return &rpcpb.Response{
 		Success: true,
@@ -292,17 +287,15 @@ func (srv *Server) handleKillEtcd() (*rpcpb.Response, error) {
 }
 
 func (srv *Server) handleFailArchive() (*rpcpb.Response, error) {
-	// TODO: stop/restart proxy?
-	// for now, just keep using the old ones
-	// if len(srv.advertisePortToProxy) > 0
+	srv.stopProxy()
 
 	// exit with stackstrace
-	srv.logger.Info("killing etcd process", zap.String("signal", syscall.SIGQUIT.String()))
+	srv.logger.Info("killing etcd", zap.String("signal", syscall.SIGQUIT.String()))
 	err := stopWithSig(srv.etcdCmd, syscall.SIGQUIT)
 	if err != nil {
 		return nil, err
 	}
-	srv.logger.Info("killed etcd process", zap.String("signal", syscall.SIGQUIT.String()))
+	srv.logger.Info("killed etcd", zap.String("signal", syscall.SIGQUIT.String()))
 
 	srv.etcdLogFile.Sync()
 	srv.etcdLogFile.Close()
@@ -336,12 +329,12 @@ func (srv *Server) handleFailArchive() (*rpcpb.Response, error) {
 
 // stop proxy, etcd, delete data directory
 func (srv *Server) handleDestroyEtcdAgent() (*rpcpb.Response, error) {
-	srv.logger.Info("killing etcd process", zap.String("signal", syscall.SIGTERM.String()))
+	srv.logger.Info("killing etcd", zap.String("signal", syscall.SIGTERM.String()))
 	err := stopWithSig(srv.etcdCmd, syscall.SIGTERM)
 	if err != nil {
 		return nil, err
 	}
-	srv.logger.Info("killed etcd process", zap.String("signal", syscall.SIGTERM.String()))
+	srv.logger.Info("killed etcd", zap.String("signal", syscall.SIGTERM.String()))
 
 	srv.logger.Info("removing base directory", zap.String("dir", srv.Member.BaseDir))
 	err = os.RemoveAll(srv.Member.BaseDir)

tools/functional-tester/cmd/etcd-tester/main.go

@@ -50,6 +50,7 @@ func main() {
 	}
 	defer clus.DestroyEtcdAgents()
 
+	logger.Info("wait health after bootstrap")
 	err = clus.WaitHealth()
 	if err != nil {
 		logger.Fatal("WaitHealth failed", zap.Error(err))

tools/functional-tester/rpcpb/rpc.pb.go

@@ -229,7 +229,7 @@ type Member struct {
 	// EtcdClientEndpoint is the etcd client endpoint.
 	EtcdClientEndpoint string `protobuf:"bytes,204,opt,name=EtcdClientEndpoint,proto3" json:"EtcdClientEndpoint,omitempty" yaml:"etcd-client-endpoint"`
 	// Etcd defines etcd binary configuration flags.
-	Etcd *Etcd `protobuf:"bytes,301,opt,name=Etcd" json:"Etcd,omitempty" yaml:"etcd-config"`
+	Etcd *Etcd `protobuf:"bytes,301,opt,name=Etcd" json:"Etcd,omitempty" yaml:"etcd"`
 }
 
 func (m *Member) Reset()                    { *m = Member{} }
@@ -258,8 +258,7 @@ type Tester struct {
 	// TODO: support no-op
 	FailureCases []string `protobuf:"bytes,31,rep,name=FailureCases" json:"FailureCases,omitempty" yaml:"failure-cases"`
 	// FailureShuffle is true to randomize failure injecting order.
-	// TODO: support shuffle
-	// bool FailureShuffle = 32 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""];
+	FailureShuffle bool `protobuf:"varint,32,opt,name=FailureShuffle,proto3" json:"FailureShuffle,omitempty" yaml:"failure-shuffle"`
 	// FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)).
 	FailpointCommands []string `protobuf:"bytes,33,rep,name=FailpointCommands" json:"FailpointCommands,omitempty" yaml:"failpoint-commands"`
 	// RunnerExecPath is a path of etcd-runner binary.
@@ -775,6 +774,18 @@ func (m *Tester) MarshalTo(dAtA []byte) (int, error) {
 			i += copy(dAtA[i:], s)
 		}
 	}
+	if m.FailureShuffle {
+		dAtA[i] = 0x80
+		i++
+		dAtA[i] = 0x2
+		i++
+		if m.FailureShuffle {
+			dAtA[i] = 1
+		} else {
+			dAtA[i] = 0
+		}
+		i++
+	}
 	if len(m.FailpointCommands) > 0 {
 		for _, s := range m.FailpointCommands {
 			dAtA[i] = 0x8a
@@ -1100,6 +1111,9 @@ func (m *Tester) Size() (n int) {
 			n += 2 + l + sovRpc(uint64(l))
 		}
 	}
+	if m.FailureShuffle {
+		n += 3
+	}
 	if len(m.FailpointCommands) > 0 {
 		for _, s := range m.FailpointCommands {
 			l = len(s)
@@ -2152,6 +2166,26 @@ func (m *Tester) Unmarshal(dAtA []byte) error {
 			}
 			m.FailureCases = append(m.FailureCases, string(dAtA[iNdEx:postIndex]))
 			iNdEx = postIndex
+		case 32:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field FailureShuffle", wireType)
+			}
+			var v int
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowRpc
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := dAtA[iNdEx]
+				iNdEx++
+				v |= (int(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
+			m.FailureShuffle = bool(v != 0)
 		case 33:
 			if wireType != 2 {
 				return fmt.Errorf("proto: wrong wireType = %d for field FailpointCommands", wireType)
@@ -2745,118 +2779,119 @@ var (
 func init() { proto.RegisterFile("rpcpb/rpc.proto", fileDescriptorRpc) }
 
 var fileDescriptorRpc = []byte{
-	// 1800 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x57, 0xcd, 0x72, 0xdb, 0xba,
-	0x15, 0xb6, 0x6c, 0x4b, 0xb6, 0x60, 0xcb, 0x66, 0x60, 0x3b, 0x66, 0x9c, 0xc4, 0xf4, 0x65, 0x7b,
-	0x33, 0xae, 0x67, 0xe8, 0x74, 0x72, 0x67, 0x3a, 0xed, 0x9d, 0xdb, 0x49, 0x25, 0x99, 0xb9, 0x76,
-	0xcd, 0x48, 0x0a, 0x24, 0x27, 0xe9, 0x4a, 0xa5, 0x48, 0x48, 0x62, 0x4d, 0x91, 0x0c, 0x08, 0xf9,
-	0x4a, 0x77, 0xd1, 0x6d, 0xb7, 0x5d, 0x76, 0xd3, 0x37, 0x68, 0x57, 0x7d, 0x89, 0xb4, 0xcd, 0xa2,
-	0x4f, 0xa0, 0xb6, 0xe9, 0x1b, 0xf0, 0x09, 0x3a, 0x00, 0x28, 0x09, 0xfa, 0x71, 0xbb, 0x33, 0xce,
-	0xf9, 0xbe, 0x8f, 0xc0, 0xd1, 0x39, 0x1f, 0x60, 0xb0, 0x4b, 0x22, 0x27, 0x6a, 0x3d, 0x27, 0x91,
-	0x73, 0x1e, 0x91, 0x90, 0x86, 0x30, 0xcb, 0x03, 0x47, 0x46, 0xc7, 0xa3, 0xdd, 0x7e, 0xeb, 0xdc,
-	0x09, 0x7b, 0xcf, 0x3b, 0x61, 0x27, 0x7c, 0xce, 0xb3, 0xad, 0x7e, 0x9b, 0xaf, 0xf8, 0x82, 0xff,
-	0x25, 0x58, 0xfa, 0xa7, 0x0d, 0xb0, 0x6e, 0x52, 0xc7, 0x85, 0x3f, 0x00, 0xeb, 0x15, 0xbb, 0x87,
-	0xd5, 0xcc, 0x49, 0xe6, 0x34, 0x5f, 0xda, 0x4d, 0x46, 0xda, 0xd6, 0xd0, 0xee, 0xf9, 0x5f, 0xeb,
-	0x81, 0xdd, 0xc3, 0x3a, 0xe2, 0x49, 0x68, 0x80, 0x8d, 0x0b, 0x9b, 0xda, 0x17, 0x1e, 0x51, 0x57,
-	0x39, 0x6e, 0x2f, 0x19, 0x69, 0xbb, 0x02, 0xe7, 0xda, 0xd4, 0x36, 0x5c, 0x8f, 0xe8, 0x68, 0x8c,
-	0x81, 0x67, 0x20, 0xf7, 0xae, 0x68, 0x31, 0xf4, 0x1a, 0x47, 0xc3, 0x64, 0xa4, 0xed, 0x08, 0xf4,
-	0x77, 0xb6, 0x2f, 0xc0, 0x29, 0x02, 0x5e, 0x01, 0xc5, 0xf2, 0x62, 0x8a, 0x83, 0xb2, 0xef, 0xe1,
-	0x80, 0xde, 0x20, 0x2b, 0x56, 0xd7, 0x4f, 0xd6, 0x4e, 0xf3, 0xa5, 0xa7, 0xc9, 0x48, 0x7b, 0x24,
-	0x58, 0x3e, 0x47, 0x18, 0x0e, 0x87, 0x18, 0x7d, 0xe2, 0xc7, 0x3a, 0x5a, 0xa0, 0x41, 0x04, 0xf6,
-	0x8a, 0xee, 0x1d, 0x26, 0xd4, 0x8b, 0xb1, 0xa4, 0x96, 0xe5, 0x6a, 0x27, 0xc9, 0x48, 0x7b, 0x22,
-	0xd4, 0xec, 0x31, 0x68, 0x56, 0x70, 0x19, 0x19, 0x96, 0xc1, 0x8e, 0xf8, 0x4e, 0x0d, 0x63, 0xc2,
-	0xe5, 0x72, 0x5c, 0xee, 0x71, 0x32, 0xd2, 0x0e, 0x67, 0x36, 0x17, 0x61, 0x4c, 0x52, 0xa5, 0x39,
-	0x0a, 0x6c, 0x01, 0xf5, 0x2a, 0xf0, 0xa8, 0x67, 0xfb, 0x93, 0x4f, 0x4c, 0xe4, 0x36, 0xb8, 0xdc,
-	0xb3, 0x64, 0xa4, 0xe9, 0x42, 0xce, 0x13, 0x48, 0x63, 0xba, 0x4b, 0x49, 0xf9, 0x5e, 0x1d, 0x58,
-	0x02, 0x3b, 0x69, 0xae, 0xec, 0xf7, 0x63, 0x8a, 0x89, 0xba, 0xc9, 0x6b, 0x7f, 0x94, 0x8c, 0xb4,
-	0x87, 0xb3, 0xca, 0x8e, 0x00, 0xe8, 0x68, 0x8e, 0xc1, 0x0a, 0x38, 0x1b, 0xa9, 0x53, 0x9b, 0x62,
-	0x35, 0xcf, 0x85, 0xa4, 0x02, 0xce, 0x09, 0x19, 0x31, 0x83, 0xe9, 0x68, 0x19, 0x79, 0x51, 0xb3,
-	0x11, 0xde, 0xe2, 0x40, 0x05, 0xff, 0x4f, 0x93, 0x32, 0xd8, 0x82, 0x26, 0x27, 0xc3, 0x97, 0xa0,
-	0x50, 0x0f, 0xec, 0x28, 0xee, 0x86, 0xb4, 0x1c, 0xf6, 0x03, 0xaa, 0x6e, 0x9d, 0x64, 0x4e, 0xd7,
-	0x4a, 0x8f, 0x92, 0x91, 0x76, 0x20, 0xd4, 0xe2, 0x34, 0x6d, 0x38, 0x2c, 0xaf, 0xa3, 0x59, 0x3c,
-	0xb4, 0xc0, 0x83, 0x37, 0xfd, 0x90, 0xda, 0x25, 0xdb, 0xb9, 0xc5, 0x81, 0x5b, 0x1a, 0x52, 0x1c,
-	0xab, 0xdb, 0x5c, 0xe4, 0x38, 0x19, 0x69, 0x47, 0x42, 0xe4, 0x03, 0x83, 0x18, 0x2d, 0x81, 0x31,
-	0x5a, 0x0c, 0xa4, 0xa3, 0x45, 0x22, 0x9b, 0x8e, 0x1a, 0xc1, 0x6f, 0x43, 0x8a, 0xd5, 0xc2, 0x49,
-	0xe6, 0x74, 0x53, 0x9e, 0x8e, 0x88, 0x60, 0xe3, 0x2e, 0x64, 0xd5, 0x19, 0x63, 0xe4, 0x8a, 0x84,
-	0x84, 0xf4, 0x23, 0x5a, 0xee, 0x62, 0xe7, 0x56, 0xdd, 0xe1, 0xd4, 0x65, 0x15, 0x11, 0x28, 0xc3,
-	0x61, 0x30, 0xa9, 0x22, 0x12, 0x59, 0xff, 0x7d, 0x16, 0xe4, 0x5e, 0xe3, 0x5e, 0x0b, 0x13, 0xf8,
-	0x73, 0xb0, 0xcd, 0x06, 0xdb, 0x1c, 0x60, 0xa7, 0x66, 0xd3, 0x6e, 0x3a, 0xd8, 0x52, 0x6d, 0x30,
-	0x75, 0x5c, 0x03, 0x0f, 0xb0, 0x63, 0x44, 0x36, 0xed, 0xea, 0x68, 0x06, 0x0e, 0xbf, 0x02, 0xf9,
-	0x62, 0x07, 0x07, 0xb4, 0xe8, 0xba, 0x84, 0xd7, 0x35, 0x5f, 0x3a, 0x48, 0x46, 0xda, 0x83, 0x74,
-	0x74, 0x58, 0xca, 0xb0, 0x5d, 0x97, 0xe8, 0x68, 0x8a, 0x63, 0xf5, 0x7c, 0x65, 0x7b, 0x7e, 0x14,
-	0x7a, 0x01, 0xbd, 0x6c, 0x34, 0x6a, 0x9c, 0xbc, 0xcd, 0xc9, 0x52, 0x3d, 0xdb, 0x63, 0x88, 0xd1,
-	0xa5, 0x34, 0x4a, 0x55, 0x16, 0x89, 0xac, 0x9e, 0x25, 0x3b, 0xc6, 0xcc, 0x3f, 0xf0, 0xbc, 0xdb,
-	0xb4, 0xec, 0x18, 0xa7, 0x6e, 0x93, 0x62, 0xe0, 0xd7, 0x60, 0x8b, 0x9d, 0xc0, 0x0a, 0x3b, 0xfc,
-	0xbc, 0x6d, 0x4e, 0x51, 0x93, 0x91, 0xb6, 0x2f, 0x9d, 0xd7, 0x0f, 0x3b, 0xe9, 0x71, 0x65, 0x30,
-	0x2c, 0x82, 0x02, 0x5b, 0x8a, 0x81, 0x6f, 0x58, 0x75, 0xf5, 0xaf, 0x19, 0xfe, 0x33, 0x48, 0x53,
-	0xc3, 0xe9, 0xa9, 0x51, 0x50, 0x36, 0x83, 0xb3, 0x0c, 0xf8, 0x2d, 0xd8, 0x9d, 0x06, 0x6a, 0x24,
-	0x1c, 0x0c, 0xd5, 0xbf, 0x09, 0x91, 0x27, 0xc9, 0x48, 0x53, 0x17, 0x45, 0x22, 0x86, 0xd1, 0xd1,
-	0x3c, 0x6b, 0xbc, 0x17, 0x36, 0xd1, 0x42, 0xe6, 0xef, 0xcb, 0xf7, 0xc2, 0xed, 0x20, 0x15, 0x99,
-	0x65, 0xc0, 0x1a, 0x80, 0x53, 0x55, 0x33, 0x70, 0x79, 0x5d, 0xd5, 0x4f, 0xa2, 0x05, 0xb4, 0x64,
-	0xa4, 0x3d, 0x5e, 0xdc, 0x0e, 0x4e, 0x61, 0x3a, 0x5a, 0xc2, 0x85, 0x3f, 0x15, 0xd7, 0x84, 0xfa,
-	0x67, 0xe6, 0xfb, 0x5b, 0x2f, 0xb6, 0xce, 0xf9, 0x6d, 0x73, 0xce, 0x62, 0xa5, 0x87, 0xc9, 0x48,
-	0x83, 0xb2, 0x60, 0x18, 0xb4, 0xbd, 0x8e, 0x8e, 0x38, 0x43, 0xff, 0x0b, 0x00, 0xb9, 0x06, 0xe6,
-	0xbe, 0xf2, 0x12, 0x14, 0xc4, 0x5f, 0x15, 0x4c, 0xbf, 0x0b, 0xc9, 0xed, 0x62, 0x4f, 0x52, 0x9e,
-	0x36, 0x02, 0x91, 0xd7, 0xd1, 0x2c, 0x1e, 0xfe, 0x04, 0x00, 0x11, 0xe0, 0x8d, 0x25, 0xae, 0x20,
-	0xe9, 0xeb, 0x29, 0x5b, 0x34, 0x94, 0x84, 0x64, 0xee, 0x7d, 0x81, 0x7d, 0x7b, 0x68, 0xd9, 0x14,
-	0x07, 0xce, 0xf0, 0x75, 0xcc, 0x3b, 0xba, 0x20, 0xbb, 0xb7, 0xcb, 0xf2, 0x86, 0x2f, 0x00, 0x46,
-	0x8f, 0xb9, 0xf7, 0x2c, 0x05, 0xfe, 0x12, 0x28, 0xb3, 0x11, 0x74, 0xc7, 0x7b, 0xbb, 0x20, 0xf7,
-	0xf6, 0xbc, 0x8c, 0x41, 0xee, 0x74, 0xb4, 0xc0, 0x63, 0x07, 0x41, 0x61, 0x3f, 0x70, 0x2d, 0xaf,
-	0xe7, 0x51, 0xf5, 0xe0, 0x24, 0x73, 0x9a, 0x95, 0x0f, 0x42, 0x58, 0xce, 0xf0, 0x59, 0x52, 0x47,
-	0x12, 0x12, 0xfe, 0x02, 0x14, 0xcc, 0x81, 0x47, 0xab, 0x01, 0x9b, 0x96, 0x3e, 0xc1, 0xea, 0xc3,
-	0x85, 0xd6, 0x18, 0x78, 0xd4, 0x08, 0x03, 0xa3, 0x2d, 0x00, 0xac, 0x35, 0x64, 0x02, 0xbc, 0x04,
-	0x4a, 0x39, 0x0c, 0x62, 0x7e, 0x31, 0x39, 0x43, 0x61, 0x39, 0x87, 0xf3, 0x6d, 0xea, 0x4c, 0x11,
-	0x63, 0xbb, 0x59, 0x60, 0xc1, 0x9f, 0x81, 0x2d, 0x33, 0xb0, 0x5b, 0x3e, 0xae, 0x45, 0x24, 0x6c,
-	0xab, 0x2a, 0x17, 0x39, 0x4c, 0x46, 0xda, 0x5e, 0xba, 0x13, 0x9e, 0x34, 0x22, 0x96, 0x65, 0xe3,
-	0x36, 0xc5, 0xc2, 0x6f, 0xc0, 0x76, 0xba, 0x9f, 0xb2, 0x1d, 0xe3, 0x58, 0xd5, 0xf8, 0xe5, 0x27,
-	0xcd, 0x6a, 0xba, 0x7b, 0xc3, 0x61, 0x69, 0x1d, 0xcd, 0xa0, 0xe1, 0xb5, 0xe4, 0x32, 0xe5, 0xb0,
-	0xd7, 0xb3, 0x03, 0x37, 0x56, 0xbf, 0x98, 0x7f, 0x2b, 0x4c, 0x5d, 0xc6, 0x49, 0x31, 0xb2, 0xc9,
-	0x8c, 0x79, 0xac, 0x35, 0x50, 0x3f, 0x08, 0x30, 0x99, 0x18, 0xe5, 0x8f, 0x78, 0x5b, 0x49, 0xad,
-	0x41, 0x78, 0x5e, 0xb6, 0xca, 0x39, 0x0a, 0x7b, 0xbc, 0x98, 0x03, 0x8a, 0x49, 0x60, 0xfb, 0x13,
-	0x99, 0x33, 0x2e, 0x23, 0x6d, 0x08, 0xa7, 0x08, 0x59, 0x68, 0x81, 0xc6, 0xaa, 0x5a, 0xa7, 0x04,
-	0xc7, 0x71, 0x63, 0x18, 0xe1, 0x58, 0xc5, 0xfc, 0x58, 0x52, 0x55, 0x63, 0x9e, 0x34, 0x28, 0xcb,
-	0xea, 0x48, 0xc6, 0xb2, 0xe6, 0x10, 0xcb, 0x6b, 0x3c, 0xac, 0x7b, 0xdf, 0x63, 0x6e, 0x81, 0x59,
-	0xb9, 0x39, 0x52, 0xf2, 0x2d, 0x1e, 0x1a, 0xb1, 0xf7, 0x3d, 0x6b, 0x8e, 0x19, 0x02, 0xf3, 0x8d,
-	0x99, 0x80, 0x65, 0x93, 0x0e, 0x56, 0x3b, 0x5c, 0x46, 0xba, 0x91, 0xe6, 0x64, 0x0c, 0x9f, 0xc1,
-	0x74, 0xb4, 0x84, 0x0b, 0xdf, 0x82, 0xfd, 0x69, 0xb4, 0xdf, 0x6e, 0x7b, 0x03, 0x64, 0x07, 0x1d,
-	0xac, 0x76, 0xb9, 0xa6, 0x9e, 0x8c, 0xb4, 0xe3, 0x45, 0x4d, 0x8e, 0x33, 0x08, 0x03, 0xea, 0x68,
-	0x29, 0x1f, 0xfe, 0x1a, 0x1c, 0x2e, 0x8b, 0x37, 0x06, 0x81, 0xea, 0x71, 0x69, 0xe9, 0x25, 0x75,
-	0x8f, 0xb4, 0x41, 0x07, 0x81, 0x8e, 0xee, 0x93, 0x61, 0x7e, 0x3e, 0x49, 0x35, 0x06, 0x41, 0x35,
-	0x8a, 0xd5, 0xdf, 0x70, 0x65, 0xe9, 0x27, 0x95, 0x94, 0xe9, 0x20, 0x30, 0xc2, 0x28, 0xd6, 0xd1,
-	0x3c, 0x8b, 0xdd, 0xa4, 0x22, 0xf4, 0xa6, 0x56, 0x57, 0x6f, 0xb9, 0x84, 0x74, 0x93, 0xa6, 0x12,
-	0x1f, 0x18, 0x75, 0x8a, 0xd3, 0x7f, 0x97, 0x01, 0x1b, 0x08, 0x7f, 0xe8, 0xe3, 0x98, 0xc2, 0x73,
-	0x90, 0xaf, 0x46, 0x98, 0xd8, 0xd4, 0x0b, 0x03, 0x6e, 0x99, 0x3b, 0x2f, 0x94, 0xd4, 0x7f, 0x27,
-	0x71, 0x34, 0x85, 0xc0, 0x2f, 0xc7, 0x6f, 0x00, 0x55, 0x98, 0x75, 0x21, 0x05, 0x8b, 0x20, 0x1a,
-	0x3f, 0x10, 0xbe, 0x1c, 0xfb, 0x32, 0x7f, 0x9d, 0x4f, 0x61, 0x22, 0x88, 0xd2, 0xa4, 0xfe, 0x0d,
-	0xd8, 0x44, 0x38, 0x8e, 0xc2, 0x20, 0xc6, 0x50, 0x05, 0x1b, 0xf5, 0xbe, 0xe3, 0xe0, 0x38, 0xe6,
-	0xfb, 0xd8, 0x44, 0xe3, 0x25, 0x7c, 0x08, 0x72, 0xec, 0x9d, 0xd7, 0x8f, 0x85, 0x2b, 0xa3, 0x74,
-	0x75, 0xf6, 0xcf, 0x8c, 0xb4, 0x79, 0xb8, 0x03, 0x40, 0x25, 0xa4, 0x75, 0x6a, 0x13, 0x8a, 0x5d,
-	0x65, 0x05, 0xee, 0x03, 0x25, 0x7d, 0xc5, 0xf0, 0x18, 0xbb, 0x2f, 0x94, 0x0c, 0xdc, 0x05, 0x5b,
-	0x08, 0xc7, 0x93, 0xc0, 0x2a, 0xdc, 0x06, 0x9b, 0xd7, 0x9e, 0xef, 0xf3, 0xd5, 0x1a, 0x4b, 0xb3,
-	0x31, 0x2e, 0x12, 0xa7, 0xeb, 0xdd, 0x61, 0x65, 0x9d, 0xa9, 0x5c, 0xe0, 0x98, 0x92, 0x70, 0xc8,
-	0x10, 0xfc, 0x35, 0xa2, 0x64, 0xe1, 0x23, 0x70, 0x50, 0xf2, 0x6d, 0xe7, 0xb6, 0x1b, 0xfa, 0xfc,
-	0x75, 0x5c, 0x0b, 0x09, 0x6d, 0x0c, 0xd0, 0x40, 0x71, 0xe1, 0x63, 0x70, 0x78, 0x13, 0xb4, 0x96,
-	0x26, 0x31, 0x3c, 0x00, 0x0f, 0xb8, 0x5d, 0xcf, 0x84, 0xdb, 0xf0, 0x10, 0xec, 0xdd, 0x04, 0xee,
-	0x42, 0xa2, 0x73, 0xf6, 0xc7, 0x35, 0xb1, 0x9f, 0xd4, 0x9e, 0x18, 0xff, 0xfa, 0xca, 0xb2, 0x9a,
-	0xd5, 0x8a, 0xd9, 0x7c, 0x55, 0xb5, 0xac, 0xea, 0x3b, 0x13, 0x29, 0x2b, 0x6c, 0xd7, 0x3c, 0x6c,
-	0x99, 0xc5, 0x0b, 0x13, 0x29, 0x19, 0x78, 0x0c, 0x8e, 0x16, 0x70, 0xcd, 0x57, 0x55, 0xd4, 0xb4,
-	0xaa, 0x95, 0x6f, 0x95, 0x55, 0xa8, 0x82, 0x7d, 0x89, 0x30, 0xcd, 0xac, 0x4d, 0xa4, 0xde, 0xdc,
-	0x54, 0xd1, 0xcd, 0x6b, 0x65, 0x9d, 0xd7, 0x87, 0x05, 0x8a, 0x96, 0xa5, 0x64, 0xe1, 0x19, 0x78,
-	0x56, 0xb2, 0x8a, 0xe5, 0xeb, 0xcb, 0xaa, 0x65, 0x36, 0x6b, 0xa6, 0x89, 0x9a, 0xb5, 0x2a, 0x6a,
-	0x34, 0x1b, 0xef, 0x9b, 0xe8, 0xfd, 0xec, 0xae, 0x72, 0xf0, 0x14, 0xfc, 0xf0, 0x7e, 0x6c, 0xfa,
-	0xe5, 0x6a, 0xc5, 0x54, 0x36, 0xe0, 0x17, 0xe0, 0xe9, 0xfd, 0x48, 0xf6, 0xe1, 0x4d, 0xf8, 0x0c,
-	0xe8, 0x17, 0xa6, 0x55, 0xfc, 0xd5, 0xff, 0xfe, 0x68, 0x1e, 0x9e, 0x80, 0x27, 0xcb, 0x71, 0x69,
-	0x6d, 0x00, 0x7c, 0x0a, 0x1e, 0x2d, 0x47, 0xb0, 0x0f, 0x6d, 0xb1, 0x36, 0x7a, 0x55, 0xbc, 0xb2,
-	0x6a, 0xd5, 0xab, 0x4a, 0xa3, 0xae, 0xb8, 0x70, 0x1b, 0x6c, 0x54, 0xaa, 0x4d, 0x16, 0x52, 0x3e,
-	0x66, 0x60, 0x01, 0x6c, 0x9a, 0xef, 0x1b, 0x26, 0xaa, 0x14, 0x2d, 0xe5, 0x4f, 0xab, 0x67, 0xbf,
-	0x05, 0x60, 0x6a, 0x92, 0x30, 0x07, 0x56, 0xaf, 0xdf, 0x2a, 0x2b, 0x30, 0x0f, 0xb2, 0x96, 0x59,
-	0xac, 0x9b, 0x0a, 0xc3, 0xe7, 0x2b, 0xd5, 0x66, 0xbd, 0x81, 0xcc, 0x7a, 0x5d, 0x59, 0x85, 0x7b,
-	0x60, 0xd7, 0xb4, 0xcc, 0x72, 0xe3, 0xaa, 0x5a, 0x69, 0xa2, 0x9b, 0x4a, 0xc5, 0x44, 0xca, 0x1a,
-	0x54, 0xc0, 0xf6, 0xbb, 0x62, 0xa3, 0x7c, 0x39, 0x8e, 0xac, 0xb3, 0x9f, 0xd9, 0xaa, 0x96, 0xaf,
-	0x9b, 0xa8, 0x58, 0x36, 0xd1, 0x38, 0x9c, 0x65, 0x40, 0xae, 0x3b, 0x8e, 0xe4, 0x5e, 0xbc, 0x04,
-	0xf9, 0x06, 0xb1, 0x83, 0x38, 0x0a, 0x09, 0x85, 0x2f, 0xe4, 0xc5, 0x4e, 0x3a, 0x70, 0xe9, 0x9c,
-	0x1f, 0xed, 0x4e, 0xd6, 0x62, 0xdc, 0xf4, 0x95, 0xd3, 0xcc, 0x8f, 0x33, 0xa5, 0xfd, 0x8f, 0xff,
-	0x3e, 0x5e, 0xf9, 0xf8, 0xf9, 0x38, 0xf3, 0x8f, 0xcf, 0xc7, 0x99, 0x7f, 0x7d, 0x3e, 0xce, 0xfc,
-	0xe1, 0x3f, 0xc7, 0x2b, 0xad, 0x1c, 0xff, 0xff, 0xfd, 0xab, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff,
-	0x53, 0xa4, 0xc8, 0x19, 0x08, 0x10, 0x00, 0x00,
+	// 1817 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x57, 0xd1, 0x72, 0xdb, 0x4a,
+	0x19, 0x8e, 0x93, 0xd8, 0x89, 0x37, 0x71, 0xa2, 0x6e, 0x92, 0x46, 0x4d, 0xdb, 0x28, 0x47, 0x70,
+	0x3a, 0x21, 0x33, 0x4a, 0xa1, 0x67, 0x86, 0x19, 0xce, 0x1c, 0xa6, 0xd8, 0x8e, 0x7a, 0x12, 0xa2,
+	0xda, 0xee, 0xda, 0x69, 0xcb, 0x95, 0x91, 0xa5, 0xb5, 0x2d, 0x22, 0x4b, 0xea, 0x6a, 0x9d, 0x63,
+	0x9f, 0x0b, 0x6e, 0x19, 0xde, 0x80, 0x1b, 0xde, 0x00, 0xde, 0xa3, 0x40, 0x2f, 0x78, 0x02, 0x03,
+	0xe5, 0x0d, 0x7c, 0xc3, 0x2d, 0xb3, 0xbb, 0xb2, 0xbd, 0xb2, 0x1d, 0xb8, 0xf3, 0xfe, 0xff, 0xf7,
+	0x7d, 0x5a, 0xfd, 0xfa, 0xff, 0x6f, 0xd7, 0x60, 0x97, 0x44, 0x4e, 0xd4, 0x7a, 0x4e, 0x22, 0xe7,
+	0x3c, 0x22, 0x21, 0x0d, 0x61, 0x96, 0x07, 0x8e, 0x8c, 0x8e, 0x47, 0xbb, 0xfd, 0xd6, 0xb9, 0x13,
+	0xf6, 0x9e, 0x77, 0xc2, 0x4e, 0xf8, 0x9c, 0x67, 0x5b, 0xfd, 0x36, 0x5f, 0xf1, 0x05, 0xff, 0x25,
+	0x58, 0xfa, 0xa7, 0x0d, 0xb0, 0x6e, 0x52, 0xc7, 0x85, 0x3f, 0x00, 0xeb, 0x15, 0xbb, 0x87, 0xd5,
+	0xcc, 0x49, 0xe6, 0x34, 0x5f, 0xda, 0x1d, 0x8f, 0xb4, 0xad, 0xa1, 0xdd, 0xf3, 0xbf, 0xd6, 0x03,
+	0xbb, 0x87, 0x75, 0xc4, 0x93, 0xd0, 0x00, 0x1b, 0x17, 0x36, 0xb5, 0x2f, 0x3c, 0xa2, 0xae, 0x72,
+	0xdc, 0xde, 0x78, 0xa4, 0xed, 0x0a, 0x9c, 0x6b, 0x53, 0xdb, 0x70, 0x3d, 0xa2, 0xa3, 0x09, 0x06,
+	0x9e, 0x81, 0xdc, 0xbb, 0xa2, 0xc5, 0xd0, 0x6b, 0x1c, 0x0d, 0xc7, 0x23, 0x6d, 0x47, 0xa0, 0xbf,
+	0xb3, 0x7d, 0x01, 0x4e, 0x10, 0xf0, 0x0a, 0x28, 0x96, 0x17, 0x53, 0x1c, 0x94, 0x7d, 0x0f, 0x07,
+	0xf4, 0x06, 0x59, 0xb1, 0xba, 0x7e, 0xb2, 0x76, 0x9a, 0x2f, 0x3d, 0x1d, 0x8f, 0xb4, 0x47, 0x82,
+	0xe5, 0x73, 0x84, 0xe1, 0x70, 0x88, 0xd1, 0x27, 0x7e, 0xac, 0xa3, 0x05, 0x1a, 0x44, 0x60, 0xaf,
+	0xe8, 0xde, 0x61, 0x42, 0xbd, 0x18, 0x4b, 0x6a, 0x59, 0xae, 0x76, 0x32, 0x1e, 0x69, 0x4f, 0x84,
+	0x9a, 0x3d, 0x01, 0xa5, 0x05, 0x97, 0x91, 0x61, 0x19, 0xec, 0x88, 0xe7, 0xd4, 0x30, 0x26, 0x5c,
+	0x2e, 0xc7, 0xe5, 0x1e, 0x8f, 0x47, 0xda, 0x61, 0x6a, 0x73, 0x11, 0xc6, 0x24, 0x51, 0x9a, 0xa3,
+	0xc0, 0x16, 0x50, 0xaf, 0x02, 0x8f, 0x7a, 0xb6, 0x3f, 0x7d, 0xc4, 0x54, 0x6e, 0x83, 0xcb, 0x3d,
+	0x1b, 0x8f, 0x34, 0x5d, 0xc8, 0x79, 0x02, 0x69, 0xcc, 0x76, 0x29, 0x29, 0xdf, 0xab, 0x03, 0x4b,
+	0x60, 0x27, 0xc9, 0x95, 0xfd, 0x7e, 0x4c, 0x31, 0x51, 0x37, 0x79, 0xed, 0x8f, 0xc6, 0x23, 0xed,
+	0x61, 0x5a, 0xd9, 0x11, 0x00, 0x1d, 0xcd, 0x31, 0x58, 0x01, 0xd3, 0x91, 0x3a, 0xb5, 0x29, 0x56,
+	0xf3, 0x5c, 0x48, 0x2a, 0xe0, 0x9c, 0x90, 0x11, 0x33, 0x98, 0x8e, 0x96, 0x91, 0x17, 0x35, 0x1b,
+	0xe1, 0x2d, 0x0e, 0x54, 0xf0, 0xff, 0x34, 0x29, 0x83, 0x2d, 0x68, 0x72, 0x32, 0x7c, 0x09, 0x0a,
+	0xf5, 0xc0, 0x8e, 0xe2, 0x6e, 0x48, 0xcb, 0x61, 0x3f, 0xa0, 0xea, 0xd6, 0x49, 0xe6, 0x74, 0xad,
+	0xf4, 0x68, 0x3c, 0xd2, 0x0e, 0x84, 0x5a, 0x9c, 0xa4, 0x0d, 0x87, 0xe5, 0x75, 0x94, 0xc6, 0x43,
+	0x0b, 0x3c, 0x78, 0xd3, 0x0f, 0xa9, 0x5d, 0xb2, 0x9d, 0x5b, 0x1c, 0xb8, 0xa5, 0x21, 0xc5, 0xb1,
+	0xba, 0xcd, 0x45, 0x8e, 0xc7, 0x23, 0xed, 0x48, 0x88, 0x7c, 0x60, 0x10, 0xa3, 0x25, 0x30, 0x46,
+	0x8b, 0x81, 0x74, 0xb4, 0x48, 0x64, 0xd3, 0x51, 0x23, 0xf8, 0x6d, 0x48, 0xb1, 0x5a, 0x38, 0xc9,
+	0x9c, 0x6e, 0xca, 0xd3, 0x11, 0x11, 0x6c, 0xdc, 0x85, 0xac, 0x3a, 0x13, 0x8c, 0x5c, 0x91, 0x90,
+	0x90, 0x7e, 0x44, 0xcb, 0x5d, 0xec, 0xdc, 0xaa, 0x3b, 0x9c, 0xba, 0xac, 0x22, 0x02, 0x65, 0x38,
+	0x0c, 0x26, 0x55, 0x44, 0x22, 0xeb, 0xbf, 0xcf, 0x82, 0xdc, 0x6b, 0xdc, 0x6b, 0x61, 0x02, 0x7f,
+	0x0e, 0xb6, 0xd9, 0x60, 0x9b, 0x03, 0xec, 0xd4, 0x6c, 0xda, 0x4d, 0x06, 0x5b, 0xaa, 0x0d, 0xa6,
+	0x8e, 0x6b, 0xe0, 0x01, 0x76, 0x8c, 0xc8, 0xa6, 0x5d, 0x1d, 0xa5, 0xe0, 0xf0, 0x2b, 0x90, 0x2f,
+	0x76, 0x70, 0x40, 0x8b, 0xae, 0x4b, 0x78, 0x5d, 0xf3, 0xa5, 0x83, 0xf1, 0x48, 0x7b, 0x90, 0x8c,
+	0x0e, 0x4b, 0x19, 0xb6, 0xeb, 0x12, 0x1d, 0xcd, 0x70, 0xac, 0x9e, 0xaf, 0x6c, 0xcf, 0x8f, 0x42,
+	0x2f, 0xa0, 0x97, 0x8d, 0x46, 0x8d, 0x93, 0xb7, 0x39, 0x59, 0xaa, 0x67, 0x7b, 0x02, 0x31, 0xba,
+	0x94, 0x46, 0x89, 0xca, 0x22, 0x91, 0xd5, 0xb3, 0x64, 0xc7, 0x98, 0xf9, 0x07, 0x9e, 0x77, 0x9b,
+	0x96, 0x1d, 0xe3, 0xc4, 0x6d, 0x12, 0x0c, 0xfc, 0x1a, 0x6c, 0xb1, 0x37, 0xb0, 0xc2, 0x0e, 0x7f,
+	0xdf, 0x36, 0xa7, 0xa8, 0xe3, 0x91, 0xb6, 0x2f, 0xbd, 0xaf, 0x1f, 0x76, 0x92, 0xd7, 0x95, 0xc1,
+	0xb0, 0x08, 0x0a, 0x6c, 0x29, 0x06, 0xbe, 0x61, 0xd5, 0xd5, 0xbf, 0x64, 0xf8, 0x67, 0x90, 0xa6,
+	0x86, 0xd3, 0x13, 0xa3, 0xa0, 0x6c, 0x06, 0xd3, 0x0c, 0xf8, 0x2d, 0xd8, 0x9d, 0x05, 0x6a, 0x24,
+	0x1c, 0x0c, 0xd5, 0xbf, 0x0a, 0x91, 0x27, 0xe3, 0x91, 0xa6, 0x2e, 0x8a, 0x44, 0x0c, 0xa3, 0xa3,
+	0x79, 0xd6, 0x64, 0x2f, 0x6c, 0xa2, 0x85, 0xcc, 0xdf, 0x96, 0xef, 0x85, 0xdb, 0x41, 0x22, 0x92,
+	0x66, 0xc0, 0x1a, 0x80, 0x33, 0x55, 0x33, 0x70, 0x79, 0x5d, 0xd5, 0x4f, 0xa2, 0x05, 0xb4, 0xf1,
+	0x48, 0x7b, 0xbc, 0xb8, 0x1d, 0x9c, 0xc0, 0x74, 0xb4, 0x84, 0x0b, 0x7f, 0x22, 0x8e, 0x09, 0xf5,
+	0xcf, 0xcc, 0xf7, 0xb7, 0x5e, 0x6c, 0x9d, 0xf3, 0xd3, 0xe6, 0x9c, 0xc5, 0xe4, 0xc3, 0x82, 0x09,
+	0xea, 0x88, 0x43, 0xf5, 0xff, 0x00, 0x90, 0x6b, 0x60, 0x6e, 0x28, 0x2f, 0x41, 0x41, 0xfc, 0xaa,
+	0x60, 0xfa, 0x5d, 0x48, 0x6e, 0x17, 0x9b, 0x91, 0xf2, 0xb4, 0x11, 0x88, 0xbc, 0x8e, 0xd2, 0x78,
+	0xf8, 0x53, 0x00, 0x44, 0x80, 0x77, 0x94, 0x38, 0x7b, 0x1e, 0x8e, 0x47, 0x1a, 0x4c, 0xb1, 0x45,
+	0x27, 0x49, 0x48, 0x66, 0xdb, 0x17, 0xd8, 0xb7, 0x87, 0x96, 0x4d, 0x71, 0xe0, 0x0c, 0x5f, 0xc7,
+	0xbc, 0x95, 0x0b, 0xb2, 0x6d, 0xbb, 0x2c, 0x6f, 0xf8, 0x02, 0x60, 0xf4, 0x98, 0x6d, 0xa7, 0x29,
+	0xf0, 0x97, 0x40, 0x49, 0x47, 0xd0, 0x1d, 0x6f, 0xea, 0x82, 0xdc, 0xd4, 0xf3, 0x32, 0x06, 0xb9,
+	0xd3, 0xd1, 0x02, 0x8f, 0xbd, 0x08, 0x0a, 0xfb, 0x81, 0x6b, 0x79, 0x3d, 0x8f, 0xaa, 0x07, 0x27,
+	0x99, 0xd3, 0xac, 0xfc, 0x22, 0x84, 0xe5, 0x0c, 0x9f, 0x25, 0x75, 0x24, 0x21, 0xe1, 0x2f, 0x40,
+	0xc1, 0x1c, 0x78, 0xb4, 0x1a, 0xb0, 0x31, 0xe9, 0x13, 0xac, 0x3e, 0x5c, 0xe8, 0x89, 0x81, 0x47,
+	0x8d, 0x30, 0x30, 0xda, 0x02, 0xc0, 0x7a, 0x42, 0x26, 0xc0, 0x4b, 0xa0, 0x94, 0xc3, 0x20, 0xe6,
+	0x27, 0x92, 0x33, 0x14, 0x5e, 0x73, 0x38, 0xdf, 0x9f, 0xce, 0x0c, 0x31, 0xf1, 0x99, 0x05, 0x16,
+	0xfc, 0x19, 0xd8, 0x32, 0x03, 0xbb, 0xe5, 0xe3, 0x5a, 0x44, 0xc2, 0xb6, 0xaa, 0x72, 0x91, 0xc3,
+	0xf1, 0x48, 0xdb, 0x4b, 0x76, 0xc2, 0x93, 0x46, 0xc4, 0xb2, 0x6c, 0xce, 0x66, 0x58, 0xf8, 0x0d,
+	0xd8, 0x4e, 0xf6, 0x53, 0xb6, 0x63, 0x1c, 0xab, 0x1a, 0x3f, 0xf5, 0xa4, 0x21, 0x4d, 0x76, 0x6f,
+	0x38, 0x2c, 0xad, 0xa3, 0x14, 0x9a, 0x9d, 0x6d, 0xc9, 0xba, 0xde, 0xed, 0xb7, 0xdb, 0x3e, 0x56,
+	0x4f, 0xe6, 0xab, 0x30, 0xe1, 0xc7, 0x02, 0xa0, 0xa3, 0x39, 0x06, 0xbc, 0x96, 0x2c, 0xaa, 0x1c,
+	0xf6, 0x7a, 0x76, 0xe0, 0xc6, 0xea, 0x17, 0xf3, 0x17, 0x8d, 0x99, 0x45, 0x39, 0x09, 0x46, 0x76,
+	0xa8, 0x09, 0x8f, 0xb5, 0x17, 0xea, 0x07, 0x01, 0x26, 0x53, 0x97, 0xfd, 0x11, 0x6f, 0x4d, 0xa9,
+	0xbd, 0x08, 0xcf, 0xcb, 0x3e, 0x3b, 0x47, 0x61, 0x37, 0x1f, 0x73, 0x40, 0x31, 0x09, 0x6c, 0x7f,
+	0x2a, 0x73, 0xc6, 0x65, 0xa4, 0x0d, 0xe1, 0x04, 0x21, 0x0b, 0x2d, 0xd0, 0xd8, 0x97, 0xa9, 0x53,
+	0x82, 0xe3, 0xb8, 0x31, 0x8c, 0x70, 0xac, 0x62, 0xfe, 0x5a, 0xd2, 0x97, 0x89, 0x79, 0xd2, 0xa0,
+	0x2c, 0xab, 0x23, 0x19, 0xcb, 0x1a, 0x4c, 0x2c, 0xaf, 0xf1, 0xb0, 0xee, 0x7d, 0x8f, 0xb9, 0x7f,
+	0x66, 0xe5, 0xd2, 0x26, 0xe4, 0x5b, 0x3c, 0x34, 0x62, 0xef, 0x7b, 0xd6, 0x60, 0x29, 0x02, 0x33,
+	0x9d, 0x54, 0xc0, 0xb2, 0x49, 0x07, 0xab, 0x1d, 0x2e, 0x23, 0x1d, 0x67, 0x73, 0x32, 0x86, 0xcf,
+	0x60, 0x3a, 0x5a, 0xc2, 0x85, 0x6f, 0xc1, 0xfe, 0x2c, 0xda, 0x6f, 0xb7, 0xbd, 0x01, 0xb2, 0x83,
+	0x0e, 0x56, 0xbb, 0x5c, 0x53, 0x1f, 0x8f, 0xb4, 0xe3, 0x45, 0x4d, 0x8e, 0x33, 0x08, 0x03, 0xea,
+	0x68, 0x29, 0x1f, 0xfe, 0x1a, 0x1c, 0x2e, 0x8b, 0x37, 0x06, 0x81, 0xea, 0x71, 0x69, 0xe9, 0x1a,
+	0x76, 0x8f, 0xb4, 0x41, 0x07, 0x81, 0x8e, 0xee, 0x93, 0x61, 0x87, 0xc1, 0x34, 0xd5, 0x18, 0x04,
+	0xd5, 0x28, 0x56, 0x7f, 0xc3, 0x95, 0xa5, 0x4f, 0x2a, 0x29, 0xd3, 0x41, 0x60, 0x84, 0x51, 0xac,
+	0xa3, 0x79, 0x16, 0x3b, 0x86, 0x45, 0xe8, 0x4d, 0xad, 0xae, 0xde, 0x72, 0x09, 0xe9, 0x18, 0x4e,
+	0x24, 0x3e, 0x30, 0xea, 0x0c, 0xa7, 0xff, 0x2e, 0x03, 0x36, 0x10, 0xfe, 0xd0, 0xc7, 0x31, 0x85,
+	0xe7, 0x20, 0x5f, 0x8d, 0x30, 0xb1, 0xa9, 0x17, 0x06, 0xdc, 0x76, 0x77, 0x5e, 0x28, 0x89, 0x79,
+	0x4f, 0xe3, 0x68, 0x06, 0x81, 0x5f, 0x4e, 0x2e, 0x10, 0xaa, 0x70, 0xfa, 0x42, 0x02, 0x16, 0x41,
+	0x34, 0xb9, 0x5d, 0x7c, 0x39, 0xf1, 0x76, 0x7e, 0xb5, 0x9f, 0xc1, 0x44, 0x10, 0x25, 0x49, 0xfd,
+	0x1b, 0xb0, 0x89, 0x70, 0x1c, 0x85, 0x41, 0x8c, 0xa1, 0x0a, 0x36, 0xea, 0x7d, 0xc7, 0xc1, 0x71,
+	0xcc, 0xf7, 0xb1, 0x89, 0x26, 0x4b, 0xf8, 0x10, 0xe4, 0xd8, 0x25, 0xb1, 0x1f, 0x0b, 0x67, 0x47,
+	0xc9, 0xea, 0xec, 0x1f, 0x19, 0x69, 0xf3, 0x70, 0x07, 0x80, 0x4a, 0x48, 0xeb, 0xd4, 0x26, 0x14,
+	0xbb, 0xca, 0x0a, 0xdc, 0x07, 0x4a, 0x72, 0x05, 0xe2, 0x31, 0x76, 0xe6, 0x28, 0x19, 0xb8, 0x0b,
+	0xb6, 0x10, 0x8e, 0xa7, 0x81, 0x55, 0xb8, 0x0d, 0x36, 0xaf, 0x3d, 0xdf, 0xe7, 0xab, 0x35, 0x96,
+	0x66, 0x63, 0x5c, 0x24, 0x4e, 0xd7, 0xbb, 0xc3, 0xca, 0x3a, 0x53, 0xb9, 0xc0, 0x31, 0x25, 0xe1,
+	0x90, 0x21, 0xf8, 0x55, 0x46, 0xc9, 0xc2, 0x47, 0xe0, 0xa0, 0xe4, 0xdb, 0xce, 0x6d, 0x37, 0xf4,
+	0xf9, 0xd5, 0xba, 0x16, 0x12, 0xda, 0x18, 0xa0, 0x81, 0xe2, 0xc2, 0xc7, 0xe0, 0xf0, 0x26, 0x68,
+	0x2d, 0x4d, 0x62, 0x78, 0x00, 0x1e, 0x70, 0xcb, 0x4f, 0x85, 0xdb, 0xf0, 0x10, 0xec, 0xdd, 0x04,
+	0xee, 0x42, 0xa2, 0x73, 0xf6, 0xc7, 0x35, 0xb1, 0x9f, 0xc4, 0xe2, 0x18, 0xff, 0xfa, 0xca, 0xb2,
+	0x9a, 0xd5, 0x8a, 0xd9, 0x7c, 0x55, 0xb5, 0xac, 0xea, 0x3b, 0x13, 0x29, 0x2b, 0x6c, 0xd7, 0x3c,
+	0x6c, 0x99, 0xc5, 0x0b, 0x13, 0x29, 0x19, 0x78, 0x0c, 0x8e, 0x16, 0x70, 0xcd, 0x57, 0x55, 0xd4,
+	0xb4, 0xaa, 0x95, 0x6f, 0x95, 0x55, 0xa8, 0x82, 0x7d, 0x89, 0x30, 0xcb, 0xac, 0x4d, 0xa5, 0xde,
+	0xdc, 0x54, 0xd1, 0xcd, 0x6b, 0x65, 0x9d, 0xd7, 0x87, 0x05, 0x8a, 0x96, 0xa5, 0x64, 0xe1, 0x19,
+	0x78, 0x56, 0xb2, 0x8a, 0xe5, 0xeb, 0xcb, 0xaa, 0x65, 0x36, 0x6b, 0xa6, 0x89, 0x9a, 0xb5, 0x2a,
+	0x6a, 0x34, 0x1b, 0xef, 0x9b, 0xe8, 0x7d, 0x7a, 0x57, 0x39, 0x78, 0x0a, 0x7e, 0x78, 0x3f, 0x36,
+	0x79, 0x72, 0xb5, 0x62, 0x2a, 0x1b, 0xf0, 0x0b, 0xf0, 0xf4, 0x7e, 0x24, 0x7b, 0xf0, 0x26, 0x7c,
+	0x06, 0xf4, 0x0b, 0xd3, 0x2a, 0xfe, 0xea, 0x7f, 0x3f, 0x34, 0x0f, 0x4f, 0xc0, 0x93, 0xe5, 0xb8,
+	0xa4, 0x36, 0x00, 0x3e, 0x05, 0x8f, 0x96, 0x23, 0xd8, 0x83, 0xb6, 0x58, 0x1b, 0xbd, 0x2a, 0x5e,
+	0x59, 0xb5, 0xea, 0x55, 0xa5, 0x51, 0x57, 0x5c, 0xb8, 0x0d, 0x36, 0x2a, 0xd5, 0x26, 0x0b, 0x29,
+	0x1f, 0x33, 0xb0, 0x00, 0x36, 0xcd, 0xf7, 0x0d, 0x13, 0x55, 0x8a, 0x96, 0xf2, 0xa7, 0xd5, 0xb3,
+	0xdf, 0x02, 0x30, 0x33, 0x49, 0x98, 0x03, 0xab, 0xd7, 0x6f, 0x95, 0x15, 0x98, 0x07, 0x59, 0xcb,
+	0x2c, 0xd6, 0x4d, 0x85, 0xe1, 0xf3, 0x95, 0x6a, 0xb3, 0xde, 0x40, 0x66, 0xbd, 0xae, 0xac, 0xc2,
+	0x3d, 0xb0, 0x6b, 0x5a, 0x66, 0xb9, 0x71, 0x55, 0xad, 0x34, 0xd1, 0x4d, 0xa5, 0x62, 0x22, 0x65,
+	0x0d, 0x2a, 0x60, 0xfb, 0x5d, 0xb1, 0x51, 0xbe, 0x9c, 0x44, 0xd6, 0xd9, 0x67, 0xb6, 0xaa, 0xe5,
+	0xeb, 0x26, 0x2a, 0x96, 0x4d, 0x34, 0x09, 0x67, 0x19, 0x90, 0xeb, 0x4e, 0x22, 0xb9, 0x17, 0x2f,
+	0x41, 0xbe, 0x41, 0xec, 0x20, 0x8e, 0x42, 0x42, 0xe1, 0x0b, 0x79, 0xb1, 0x93, 0x0c, 0x5c, 0x32,
+	0xe7, 0x47, 0xbb, 0xd3, 0xb5, 0x18, 0x37, 0x7d, 0xe5, 0x34, 0xf3, 0xe3, 0x4c, 0x69, 0xff, 0xe3,
+	0xbf, 0x8e, 0x57, 0x3e, 0x7e, 0x3e, 0xce, 0xfc, 0xfd, 0xf3, 0x71, 0xe6, 0x9f, 0x9f, 0x8f, 0x33,
+	0x7f, 0xf8, 0xf7, 0xf1, 0x4a, 0x2b, 0xc7, 0xff, 0xfc, 0x7f, 0xf5, 0xdf, 0x00, 0x00, 0x00, 0xff,
+	0xff, 0xd3, 0x41, 0x5a, 0x89, 0x45, 0x10, 0x00, 0x00,
 }

tools/functional-tester/rpcpb/rpc.proto

@@ -81,7 +81,7 @@ message Member {
   string EtcdClientEndpoint = 204 [(gogoproto.moretags) = "yaml:\"etcd-client-endpoint\""];
 
   // Etcd defines etcd binary configuration flags.
-  Etcd Etcd = 301 [(gogoproto.moretags) = "yaml:\"etcd-config\""];
+  Etcd Etcd = 301 [(gogoproto.moretags) = "yaml:\"etcd\""];
 }
 
 enum FailureCase {
@@ -143,8 +143,7 @@ message Tester {
   // TODO: support no-op
   repeated string FailureCases = 31 [(gogoproto.moretags) = "yaml:\"failure-cases\""];
   // FailureShuffle is true to randomize failure injecting order.
-  // TODO: support shuffle
-  // bool FailureShuffle = 32 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""];
+  bool FailureShuffle = 32 [(gogoproto.moretags) = "yaml:\"failure-shuffle\""];
   // FailpointCommands is the list of "gofail" commands (e.g. panic("etcd-tester"),1*sleep(1000)).
   repeated string FailpointCommands = 33 [(gogoproto.moretags) = "yaml:\"failpoint-commands\""];
 

tools/functional-tester/tester/cluster.go

@@ -19,6 +19,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"math/rand"
 	"net/http"
 	"path/filepath"
 	"strings"
@@ -27,10 +28,10 @@ import (
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/pkg/debugutil"
 	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
-	"golang.org/x/time/rate"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.uber.org/zap"
+	"golang.org/x/time/rate"
 	"google.golang.org/grpc"
 	yaml "gopkg.in/yaml.v2"
 )
@@ -234,6 +235,33 @@ func NewCluster(logger *zap.Logger, fpath string) (*Cluster, error) {
 	}
 	go clus.serveTesterServer()
 
+	clus.updateFailures()
+
+	clus.rateLimiter = rate.NewLimiter(
+		rate.Limit(int(clus.Tester.StressQPS)),
+		int(clus.Tester.StressQPS),
+	)
+	clus.updateStresserChecker()
+	return clus, nil
+}
+
+func (clus *Cluster) serveTesterServer() {
+	clus.logger.Info(
+		"started tester HTTP server",
+		zap.String("tester-address", clus.Tester.TesterAddr),
+	)
+	err := clus.testerHTTPServer.ListenAndServe()
+	clus.logger.Info(
+		"tester HTTP server returned",
+		zap.String("tester-address", clus.Tester.TesterAddr),
+		zap.Error(err),
+	)
+	if err != nil && err != http.ErrServerClosed {
+		clus.logger.Fatal("tester HTTP errored", zap.Error(err))
+	}
+}
+
+func (clus *Cluster) updateFailures() {
 	for _, cs := range clus.Tester.FailureCases {
 		switch cs {
 		case "KILL_ONE_FOLLOWER":
@@ -270,33 +298,59 @@ func NewCluster(logger *zap.Logger, fpath string) (*Cluster, error) {
 			clus.failures = append(clus.failures, newFailureNoOp())
 		case "EXTERNAL":
 			clus.failures = append(clus.failures, newFailureExternal(clus.Tester.ExternalExecPath))
-		default:
-			return nil, fmt.Errorf("unknown failure %q", cs)
 		}
 	}
+}
 
-	clus.rateLimiter = rate.NewLimiter(
-		rate.Limit(int(clus.Tester.StressQPS)),
-		int(clus.Tester.StressQPS),
-	)
-	clus.updateStresserChecker()
-	return clus, nil
+func (clus *Cluster) failureStrings() (fs []string) {
+	fs = make([]string, len(clus.failures))
+	for i := range clus.failures {
+		fs[i] = clus.failures[i].Desc()
+	}
+	return fs
 }
 
-func (clus *Cluster) serveTesterServer() {
-	clus.logger.Info(
-		"started tester HTTP server",
-		zap.String("tester-address", clus.Tester.TesterAddr),
-	)
-	err := clus.testerHTTPServer.ListenAndServe()
-	clus.logger.Info(
-		"tester HTTP server returned",
-		zap.String("tester-address", clus.Tester.TesterAddr),
-		zap.Error(err),
-	)
-	if err != nil && err != http.ErrServerClosed {
-		clus.logger.Fatal("tester HTTP errored", zap.Error(err))
+func (clus *Cluster) shuffleFailures() {
+	rand.Seed(time.Now().UnixNano())
+	offset := rand.Intn(1000)
+	n := len(clus.failures)
+	cp := coprime(n)
+
+	clus.logger.Info("shuffling test failure cases", zap.Int("total", n))
+	fs := make([]Failure, n)
+	for i := 0; i < n; i++ {
+		fs[i] = clus.failures[(cp*i+offset)%n]
+	}
+	clus.failures = fs
+	clus.logger.Info("shuffled test failure cases", zap.Int("total", n))
+}
+
+/*
+x and y of GCD 1 are coprime to each other
+
+x1 = ( coprime of n * idx1 + offset ) % n
+x2 = ( coprime of n * idx2 + offset ) % n
+(x2 - x1) = coprime of n * (idx2 - idx1) % n
+          = (idx2 - idx1) = 1
+
+Consecutive x's are guaranteed to be distinct
+*/
+func coprime(n int) int {
+	coprime := 1
+	for i := n / 2; i < n; i++ {
+		if gcd(i, n) == 1 {
+			coprime = i
+			break
+		}
 	}
+	return coprime
+}
+
+func gcd(x, y int) int {
+	if y == 0 {
+		return x
+	}
+	return gcd(y, x%y)
 }
 
 func (clus *Cluster) updateStresserChecker() {
@@ -522,8 +576,6 @@ func (clus *Cluster) DestroyEtcdAgents() {
 		clus.logger.Info("closed connection to agent", zap.String("agent-address", clus.Members[i].AgentAddr), zap.Error(err))
 	}
 
-	// TODO: closing stresser connections to etcd
-
 	if clus.testerHTTPServer != nil {
 		clus.logger.Info("closing tester HTTP server", zap.String("tester-address", clus.Tester.TesterAddr))
 		ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
@@ -558,7 +610,7 @@ func (clus *Cluster) WaitHealth() error {
 				break
 			}
 			clus.logger.Info(
-				"successfully wrote health key",
+				"wrote health key",
 				zap.Int("retries", i),
 				zap.String("endpoint", m.EtcdClientEndpoint),
 			)
@@ -642,9 +694,9 @@ func (clus *Cluster) compactKV(rev int64, timeout time.Duration) (err error) {
 		kvc := pb.NewKVClient(conn)
 
 		clus.logger.Info(
-			"starting compaction",
+			"compacting",
 			zap.String("endpoint", m.EtcdClientEndpoint),
-			zap.Int64("revision", rev),
+			zap.Int64("compact-revision", rev),
 			zap.Duration("timeout", timeout),
 		)
 
@@ -660,14 +712,14 @@ func (clus *Cluster) compactKV(rev int64, timeout time.Duration) (err error) {
 				clus.logger.Info(
 					"compact error is ignored",
 					zap.String("endpoint", m.EtcdClientEndpoint),
-					zap.Int64("revision", rev),
+					zap.Int64("compact-revision", rev),
 					zap.Error(cerr),
 				)
 			} else {
 				clus.logger.Warn(
 					"compact failed",
 					zap.String("endpoint", m.EtcdClientEndpoint),
-					zap.Int64("revision", rev),
+					zap.Int64("compact-revision", rev),
 					zap.Error(cerr),
 				)
 				err = cerr
@@ -677,9 +729,9 @@ func (clus *Cluster) compactKV(rev int64, timeout time.Duration) (err error) {
 
 		if succeed {
 			clus.logger.Info(
-				"finished compaction",
+				"compacted",
 				zap.String("endpoint", m.EtcdClientEndpoint),
-				zap.Int64("revision", rev),
+				zap.Int64("compact-revision", rev),
 				zap.Duration("timeout", timeout),
 				zap.Duration("took", time.Since(now)),
 			)

tools/functional-tester/tester/cluster_test.go

@@ -16,6 +16,7 @@ package tester
 
 import (
 	"reflect"
+	"sort"
 	"testing"
 
 	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
@@ -131,6 +132,7 @@ func Test_newCluster(t *testing.T) {
 				"DELAY_PEER_PORT_TX_RX_LEADER",
 				"DELAY_PEER_PORT_TX_RX_ALL",
 			},
+			FailureShuffle:          true,
 			FailpointCommands:       []string{`panic("etcd-tester")`},
 			RunnerExecPath:          "/etcd-runner",
 			ExternalExecPath:        "",
@@ -159,4 +161,33 @@ func Test_newCluster(t *testing.T) {
 	if !reflect.DeepEqual(exp, cfg) {
 		t.Fatalf("expected %+v, got %+v", exp, cfg)
 	}
+
+	cfg.logger = logger
+
+	cfg.updateFailures()
+	fs1 := cfg.failureStrings()
+
+	cfg.shuffleFailures()
+	fs2 := cfg.failureStrings()
+	if reflect.DeepEqual(fs1, fs2) {
+		t.Fatalf("expected shuffled failure cases, got %q", fs2)
+	}
+
+	cfg.shuffleFailures()
+	fs3 := cfg.failureStrings()
+	if reflect.DeepEqual(fs2, fs3) {
+		t.Fatalf("expected reshuffled failure cases from %q, got %q", fs2, fs3)
+	}
+
+	// shuffle ensures visit all exactly once
+	// so when sorted, failure cases must be equal
+	sort.Strings(fs1)
+	sort.Strings(fs2)
+	sort.Strings(fs3)
+	if !reflect.DeepEqual(fs1, fs2) {
+		t.Fatalf("expected %q, got %q", fs1, fs2)
+	}
+	if !reflect.DeepEqual(fs2, fs3) {
+		t.Fatalf("expected %q, got %q", fs2, fs3)
+	}
 }

tools/functional-tester/tester/tester.go → tools/functional-tester/tester/cluster_tester.go

@@ -34,9 +34,9 @@ func (clus *Cluster) StartTester() {
 	var preModifiedKey int64
 	for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ {
 		roundTotalCounter.Inc()
-
 		clus.rd = round
-		if err := clus.doRound(round); err != nil {
+
+		if err := clus.doRound(); err != nil {
 			clus.logger.Warn(
 				"doRound failed; returning",
 				zap.Int("round", clus.rd),
@@ -50,6 +50,7 @@ func (clus *Cluster) StartTester() {
 			preModifiedKey = 0
 			continue
 		}
+
 		// -1 so that logPrefix doesn't print out 'case'
 		clus.cs = -1
 
@@ -99,18 +100,28 @@ func (clus *Cluster) StartTester() {
 	}
 
 	clus.logger.Info(
-		"functional-tester is finished",
+		"functional-tester passed",
 		zap.Int("round", clus.rd),
 		zap.Int("case", clus.cs),
 	)
 }
 
-func (clus *Cluster) doRound(round int) error {
+func (clus *Cluster) doRound() error {
+	if clus.Tester.FailureShuffle {
+		clus.shuffleFailures()
+	}
+
+	clus.logger.Info(
+		"starting round",
+		zap.Int("round", clus.rd),
+		zap.Strings("failures", clus.failureStrings()),
+	)
 	for i, f := range clus.failures {
 		clus.cs = i
 
 		caseTotalCounter.WithLabelValues(f.Desc()).Inc()
 
+		clus.logger.Info("wait health before injecting failures")
 		if err := clus.WaitHealth(); err != nil {
 			return fmt.Errorf("wait full health error: %v", err)
 		}
@@ -121,7 +132,7 @@ func (clus *Cluster) doRound(round int) error {
 			zap.Int("case", clus.cs),
 			zap.String("desc", f.Desc()),
 		)
-		if err := f.Inject(clus, round); err != nil {
+		if err := f.Inject(clus); err != nil {
 			return fmt.Errorf("injection error: %v", err)
 		}
 		clus.logger.Info(
@@ -131,13 +142,16 @@ func (clus *Cluster) doRound(round int) error {
 			zap.String("desc", f.Desc()),
 		)
 
+		// if run local, recovering server may conflict
+		// with stressing client ports
+		// TODO: use unix for local tests
 		clus.logger.Info(
 			"recovering failure",
 			zap.Int("round", clus.rd),
 			zap.Int("case", clus.cs),
 			zap.String("desc", f.Desc()),
 		)
-		if err := f.Recover(clus, round); err != nil {
+		if err := f.Recover(clus); err != nil {
 			return fmt.Errorf("recovery error: %v", err)
 		}
 		clus.logger.Info(
@@ -147,22 +161,31 @@ func (clus *Cluster) doRound(round int) error {
 			zap.String("desc", f.Desc()),
 		)
 
+		clus.logger.Info("pausing stresser after failure recovery, before wait health")
 		clus.pauseStresser()
 
+		clus.logger.Info("wait health after recovering failures")
 		if err := clus.WaitHealth(); err != nil {
 			return fmt.Errorf("wait full health error: %v", err)
 		}
+		clus.logger.Info("check consistency after recovering failures")
 		if err := clus.checkConsistency(); err != nil {
 			return fmt.Errorf("tt.checkConsistency error (%v)", err)
 		}
 
 		clus.logger.Info(
-			"success",
+			"failure case passed",
 			zap.Int("round", clus.rd),
 			zap.Int("case", clus.cs),
 			zap.String("desc", f.Desc()),
 		)
 	}
+
+	clus.logger.Info(
+		"finished round",
+		zap.Int("round", clus.rd),
+		zap.Strings("failures", clus.failureStrings()),
+	)
 	return nil
 }
 
@@ -181,6 +204,7 @@ func (clus *Cluster) updateRevision() error {
 }
 
 func (clus *Cluster) compact(rev int64, timeout time.Duration) (err error) {
+	clus.logger.Info("pausing stresser before compact")
 	clus.pauseStresser()
 	defer func() {
 		if err == nil {
@@ -252,7 +276,7 @@ func (clus *Cluster) cleanup() error {
 	clus.closeStresser()
 	if err := clus.FailArchive(); err != nil {
 		clus.logger.Warn(
-			"Cleanup failed",
+			"cleanup failed",
 			zap.Int("round", clus.rd),
 			zap.Int("case", clus.cs),
 			zap.Error(err),
@@ -261,7 +285,7 @@ func (clus *Cluster) cleanup() error {
 	}
 	if err := clus.Restart(); err != nil {
 		clus.logger.Warn(
-			"Restart failed",
+			"restart failed",
 			zap.Int("round", clus.rd),
 			zap.Int("case", clus.cs),
 			zap.Error(err),

tools/functional-tester/tester/failure.go

@@ -21,10 +21,10 @@ package tester
 type Failure interface {
 	// Inject injeccts the failure into the testing cluster at the given
 	// round. When calling the function, the cluster should be in health.
-	Inject(clus *Cluster, round int) error
+	Inject(clus *Cluster) error
 	// Recover recovers the injected failure caused by the injection of the
 	// given round and wait for the recovery of the testing cluster.
-	Recover(clus *Cluster, round int) error
+	Recover(clus *Cluster) error
 	// Desc returns a description of the failure
 	Desc() string
 }

tools/functional-tester/tester/failure_case_delay.go

@@ -25,8 +25,8 @@ type failureDelay struct {
 	delayDuration time.Duration
 }
 
-func (f *failureDelay) Inject(clus *Cluster, round int) error {
-	if err := f.Failure.Inject(clus, round); err != nil {
+func (f *failureDelay) Inject(clus *Cluster) error {
+	if err := f.Failure.Inject(clus); err != nil {
 		return err
 	}
 	if f.delayDuration > 0 {

tools/functional-tester/tester/failure_case_external.go

@@ -26,12 +26,12 @@ type failureExternal struct {
 	scriptPath  string
 }
 
-func (f *failureExternal) Inject(clus *Cluster, round int) error {
-	return exec.Command(f.scriptPath, "enable", fmt.Sprintf("%d", round)).Run()
+func (f *failureExternal) Inject(clus *Cluster) error {
+	return exec.Command(f.scriptPath, "enable", fmt.Sprintf("%d", clus.rd)).Run()
 }
 
-func (f *failureExternal) Recover(clus *Cluster, round int) error {
-	return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", round)).Run()
+func (f *failureExternal) Recover(clus *Cluster) error {
+	return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", clus.rd)).Run()
 }
 
 func (f *failureExternal) Desc() string { return f.description }

tools/functional-tester/tester/failure_case_kill.go

@@ -99,18 +99,19 @@ type failureLeader struct {
 // failureUntilSnapshot injects a failure and waits for a snapshot event
 type failureUntilSnapshot struct{ Failure }
 
-func (f *failureOne) Inject(clus *Cluster, round int) error {
-	return f.injectMember(clus, round%len(clus.Members))
+func (f *failureOne) Inject(clus *Cluster) error {
+	return f.injectMember(clus, clus.rd%len(clus.Members))
 }
 
-func (f *failureOne) Recover(clus *Cluster, round int) error {
-	if err := f.recoverMember(clus, round%len(clus.Members)); err != nil {
+func (f *failureOne) Recover(clus *Cluster) error {
+	if err := f.recoverMember(clus, clus.rd%len(clus.Members)); err != nil {
 		return err
 	}
+	clus.logger.Info("wait health after recovering failureOne")
 	return clus.WaitHealth()
 }
 
-func (f *failureAll) Inject(clus *Cluster, round int) error {
+func (f *failureAll) Inject(clus *Cluster) error {
 	for i := range clus.Members {
 		if err := f.injectMember(clus, i); err != nil {
 			return err
@@ -119,17 +120,18 @@ func (f *failureAll) Inject(clus *Cluster, round int) error {
 	return nil
 }
 
-func (f *failureAll) Recover(clus *Cluster, round int) error {
+func (f *failureAll) Recover(clus *Cluster) error {
 	for i := range clus.Members {
 		if err := f.recoverMember(clus, i); err != nil {
 			return err
 		}
 	}
+	clus.logger.Info("wait health after recovering failureAll")
 	return clus.WaitHealth()
 }
 
-func (f *failureQuorum) Inject(clus *Cluster, round int) error {
-	for i := range killMap(len(clus.Members), round) {
+func (f *failureQuorum) Inject(clus *Cluster) error {
+	for i := range killMap(len(clus.Members), clus.rd) {
 		if err := f.injectMember(clus, i); err != nil {
 			return err
 		}
@@ -137,8 +139,8 @@ func (f *failureQuorum) Inject(clus *Cluster, round int) error {
 	return nil
 }
 
-func (f *failureQuorum) Recover(clus *Cluster, round int) error {
-	for i := range killMap(len(clus.Members), round) {
+func (f *failureQuorum) Recover(clus *Cluster) error {
+	for i := range killMap(len(clus.Members), clus.rd) {
 		if err := f.recoverMember(clus, i); err != nil {
 			return err
 		}
@@ -146,7 +148,7 @@ func (f *failureQuorum) Recover(clus *Cluster, round int) error {
 	return nil
 }
 
-func (f *failureLeader) Inject(clus *Cluster, round int) error {
+func (f *failureLeader) Inject(clus *Cluster) error {
 	idx, err := clus.GetLeader()
 	if err != nil {
 		return err
@@ -155,15 +157,16 @@ func (f *failureLeader) Inject(clus *Cluster, round int) error {
 	return f.injectMember(clus, idx)
 }
 
-func (f *failureLeader) Recover(clus *Cluster, round int) error {
+func (f *failureLeader) Recover(clus *Cluster) error {
 	if err := f.recoverMember(clus, f.idx); err != nil {
 		return err
 	}
+	clus.logger.Info("wait health after recovering failureLeader")
 	return clus.WaitHealth()
 }
 
-func (f *failureUntilSnapshot) Inject(clus *Cluster, round int) error {
-	if err := f.Failure.Inject(clus, round); err != nil {
+func (f *failureUntilSnapshot) Inject(clus *Cluster) error {
+	if err := f.Failure.Inject(clus); err != nil {
 		return err
 	}
 	if len(clus.Members) < 3 {

tools/functional-tester/tester/failure_case_no_op.go

@@ -16,8 +16,8 @@ package tester
 
 type failureNoOp failureByFunc
 
-func (f *failureNoOp) Inject(clus *Cluster, round int) error  { return nil }
-func (f *failureNoOp) Recover(clus *Cluster, round int) error { return nil }
+func (f *failureNoOp) Inject(clus *Cluster) error  { return nil }
+func (f *failureNoOp) Recover(clus *Cluster) error { return nil }
 
 func newFailureNoOp() Failure {
 	return &failureNoOp{

tools/functional-tester/tester/local-test.yaml

@@ -8,7 +8,7 @@ agent-configs:
   etcd-client-proxy: false
   etcd-peer-proxy: true
   etcd-client-endpoint: 127.0.0.1:1379
-  etcd-config:
+  etcd:
     name: s1
     data-dir: /tmp/etcd-agent-data-1/etcd.data
     wal-dir: /tmp/etcd-agent-data-1/etcd.data/member/wal
@@ -32,7 +32,7 @@ agent-configs:
   etcd-client-proxy: false
   etcd-peer-proxy: true
   etcd-client-endpoint: 127.0.0.1:2379
-  etcd-config:
+  etcd:
     name: s2
     data-dir: /tmp/etcd-agent-data-2/etcd.data
     wal-dir: /tmp/etcd-agent-data-2/etcd.data/member/wal
@@ -56,7 +56,7 @@ agent-configs:
   etcd-client-proxy: false
   etcd-peer-proxy: true
   etcd-client-endpoint: 127.0.0.1:3379
-  etcd-config:
+  etcd:
     name: s3
     data-dir: /tmp/etcd-agent-data-3/etcd.data
     wal-dir: /tmp/etcd-agent-data-3/etcd.data/member/wal
@@ -98,9 +98,7 @@ tester-config:
   - DELAY_PEER_PORT_TX_RX_LEADER
   - DELAY_PEER_PORT_TX_RX_ALL
 
-  # TODO: shuffle
-  # fail-shuffle: true
-
+  failure-shuffle: true
   failpoint-commands:
   - panic("etcd-tester")
   # failpoint-commands:

tools/functional-tester/tester/stress.go

@@ -16,7 +16,6 @@ package tester
 
 import (
 	"fmt"
-	"sync"
 	"time"
 
 	"go.uber.org/zap"
@@ -35,82 +34,6 @@ type Stresser interface {
 	Checker() Checker
 }
 
-// nopStresser implements Stresser that does nothing
-type nopStresser struct {
-	start time.Time
-	qps   int
-}
-
-func (s *nopStresser) Stress() error { return nil }
-func (s *nopStresser) Pause()        {}
-func (s *nopStresser) Close()        {}
-func (s *nopStresser) ModifiedKeys() int64 {
-	return 0
-}
-func (s *nopStresser) Checker() Checker { return nil }
-
-// compositeStresser implements a Stresser that runs a slice of
-// stressing clients concurrently.
-type compositeStresser struct {
-	stressers []Stresser
-}
-
-func (cs *compositeStresser) Stress() error {
-	for i, s := range cs.stressers {
-		if err := s.Stress(); err != nil {
-			for j := 0; j < i; j++ {
-				cs.stressers[i].Close()
-			}
-			return err
-		}
-	}
-	return nil
-}
-
-func (cs *compositeStresser) Pause() {
-	var wg sync.WaitGroup
-	wg.Add(len(cs.stressers))
-	for i := range cs.stressers {
-		go func(s Stresser) {
-			defer wg.Done()
-			s.Pause()
-		}(cs.stressers[i])
-	}
-	wg.Wait()
-}
-
-func (cs *compositeStresser) Close() {
-	var wg sync.WaitGroup
-	wg.Add(len(cs.stressers))
-	for i := range cs.stressers {
-		go func(s Stresser) {
-			defer wg.Done()
-			s.Close()
-		}(cs.stressers[i])
-	}
-	wg.Wait()
-}
-
-func (cs *compositeStresser) ModifiedKeys() (modifiedKey int64) {
-	for _, stress := range cs.stressers {
-		modifiedKey += stress.ModifiedKeys()
-	}
-	return modifiedKey
-}
-
-func (cs *compositeStresser) Checker() Checker {
-	var chks []Checker
-	for _, s := range cs.stressers {
-		if chk := s.Checker(); chk != nil {
-			chks = append(chks, chk)
-		}
-	}
-	if len(chks) == 0 {
-		return nil
-	}
-	return newCompositeChecker(chks)
-}
-
 // newStresser creates stresser from a comma separated list of stresser types.
 func newStresser(clus *Cluster, idx int) Stresser {
 	stressers := make([]Stresser, len(clus.Tester.StressTypes))

tools/functional-tester/tester/stress_composite.go

@@ -0,0 +1,79 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tester
+
+import "sync"
+
+// compositeStresser implements a Stresser that runs a slice of
+// stressing clients concurrently.
+type compositeStresser struct {
+	stressers []Stresser
+}
+
+func (cs *compositeStresser) Stress() error {
+	for i, s := range cs.stressers {
+		if err := s.Stress(); err != nil {
+			for j := 0; j < i; j++ {
+				cs.stressers[j].Close()
+			}
+			return err
+		}
+	}
+	return nil
+}
+
+func (cs *compositeStresser) Pause() {
+	var wg sync.WaitGroup
+	wg.Add(len(cs.stressers))
+	for i := range cs.stressers {
+		go func(s Stresser) {
+			defer wg.Done()
+			s.Pause()
+		}(cs.stressers[i])
+	}
+	wg.Wait()
+}
+
+func (cs *compositeStresser) Close() {
+	var wg sync.WaitGroup
+	wg.Add(len(cs.stressers))
+	for i := range cs.stressers {
+		go func(s Stresser) {
+			defer wg.Done()
+			s.Close()
+		}(cs.stressers[i])
+	}
+	wg.Wait()
+}
+
+func (cs *compositeStresser) ModifiedKeys() (modifiedKey int64) {
+	for _, stress := range cs.stressers {
+		modifiedKey += stress.ModifiedKeys()
+	}
+	return modifiedKey
+}
+
+func (cs *compositeStresser) Checker() Checker {
+	var chks []Checker
+	for _, s := range cs.stressers {
+		if chk := s.Checker(); chk != nil {
+			chks = append(chks, chk)
+		}
+	}
+	if len(chks) == 0 {
+		return nil
+	}
+	return newCompositeChecker(chks)
+}

tools/functional-tester/tester/stress_nop.go

@@ -0,0 +1,31 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tester
+
+import "time"
+
+// nopStresser implements Stresser that does nothing
+type nopStresser struct {
+	start time.Time
+	qps   int
+}
+
+func (s *nopStresser) Stress() error { return nil }
+func (s *nopStresser) Pause()        {}
+func (s *nopStresser) Close()        {}
+func (s *nopStresser) ModifiedKeys() int64 {
+	return 0
+}
+func (s *nopStresser) Checker() Checker { return nil }