Merge pull request #9534 from gyuho/test-tls

functional-tester: enable TLS, phase 1

CHANGELOG-3.4.md

@@ -30,7 +30,7 @@ See [code changes](https://github.com/coreos/etcd/compare/v3.3.0...v3.4.0) and [
   - Futhermore, when `--auto-compaction-mode=periodic --auto-compaction-retention=30m` and writes per minute are about 1000, `v3.3.0`, `v3.3.1`, and `v3.3.2` compact revision 30000, 33000, and 36000, for every 3-minute, while `v3.3.3` *or later* compacts revision 30000, 60000, and 90000, for every 30-minute.
 - Improve [lease expire/revoke operation performance](https://github.com/coreos/etcd/pull/9418), address [lease scalability issue](https://github.com/coreos/etcd/issues/9496).
 - Make [Lease `Lookup` non-blocking with concurrent `Grant`/`Revoke`](https://github.com/coreos/etcd/pull/9229).
-- Improve functional tester coverage: use [proxy layer to run network fault tests in CIs](https://github.com/coreos/etcd/pull/9081), enable [TLS](https://github.com/coreos/etcd/issues/8943), add [liveness mode](https://github.com/coreos/etcd/issues/9230), [shuffle test sequence](https://github.com/coreos/etcd/issues/9381).
+- Improve [functional tester](https://github.com/coreos/etcd/tree/master/tools/functional-tester) coverage: use [proxy layer to run network fault tests in CI](https://github.com/coreos/etcd/pull/9081), enable [TLS both for server and client](https://github.com/coreos/etcd/pull/9534), add [liveness mode](https://github.com/coreos/etcd/issues/9230), and [shuffle test sequence](https://github.com/coreos/etcd/issues/9381).
 
 ### Breaking Changes
 

tools/functional-tester/README.md

@@ -2,19 +2,7 @@
 
 etcd functional test suite tests the functionality of an etcd cluster with a focus on failure resistance under high pressure. It sets up an etcd cluster and inject failures into the cluster by killing the process or isolate the network of the process. It expects the etcd cluster to recover within a short amount of time after fixing the fault.
 
-etcd functional test suite has two components: etcd-agent and etcd-tester. etcd-agent runs on every test machines and etcd-tester is a single controller of the test. etcd-tester controls all the etcd-agent to start etcd clusters and simulate various failure cases.
-
-## Requirements
-
-The environment of the cluster must be stable enough, so etcd test suite can assume that most of the failures are generated by itself.
-
-## etcd agent
-
-etcd agent is a daemon on each machines. It can start, stop, restart, isolate and terminate an etcd process. The agent exposes these functionality via HTTP RPC.
-
-## etcd tester
-
-etcd functional tester control the progress of the functional tests. It calls the RPC of the etcd agent to simulate various test cases. For example, it can start a three members cluster by sending three start RPC calls to three different etcd agents. It can make one of the member failed by sending stop RPC call to one etcd agent.
+etcd functional test suite has two components: etcd-agent and etcd-tester. etcd-agent runs on every test machine, and etcd-tester is a single controller of the test. tester controls agents: start etcd process, stop, terminate, inject failures, and so on.
 
 ### Run locally
 

tools/functional-tester/agent/handler.go

@@ -17,9 +17,11 @@ package agent
 import (
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"net/url"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"syscall"
 	"time"
 
@@ -72,6 +74,7 @@ func (srv *Server) handleInitialStartEtcd(req *rpcpb.Request) (*rpcpb.Response,
 		return &rpcpb.Response{
 			Success: false,
 			Status:  fmt.Sprintf("%q is not valid; last server operation was %q", rpcpb.Operation_InitialStartEtcd.String(), srv.last.String()),
+			Member:  req.Member,
 		}, nil
 	}
 
@@ -84,16 +87,22 @@ func (srv *Server) handleInitialStartEtcd(req *rpcpb.Request) (*rpcpb.Response,
 	}
 	srv.lg.Info("created base directory", zap.String("path", srv.Member.BaseDir))
 
-	if err = srv.createEtcdFile(); err != nil {
+	if err = srv.saveEtcdLogFile(); err != nil {
 		return nil, err
 	}
+
 	srv.creatEtcdCmd()
 
-	err = srv.startEtcdCmd()
-	if err != nil {
+	if err = srv.saveTLSAssets(); err != nil {
+		return nil, err
+	}
+	if err = srv.startEtcdCmd(); err != nil {
 		return nil, err
 	}
 	srv.lg.Info("started etcd", zap.String("command-path", srv.etcdCmd.Path))
+	if err = srv.loadAutoTLSAssets(); err != nil {
+		return nil, err
+	}
 
 	// wait some time for etcd listener start
 	// before setting up proxy
@@ -104,10 +113,12 @@ func (srv *Server) handleInitialStartEtcd(req *rpcpb.Request) (*rpcpb.Response,
 
 	return &rpcpb.Response{
 		Success: true,
-		Status:  "successfully started etcd!",
+		Status:  "start etcd PASS",
+		Member:  srv.Member,
 	}, nil
 }
 
+// TODO: support TLS
 func (srv *Server) startProxy() error {
 	if srv.Member.EtcdClientProxy {
 		advertiseClientURL, advertiseClientURLPort, err := getURLAndPort(srv.Member.Etcd.AdvertiseClientURLs[0])
@@ -133,7 +144,7 @@ func (srv *Server) startProxy() error {
 	}
 
 	if srv.Member.EtcdPeerProxy {
-		advertisePeerURL, advertisePeerURLPort, err := getURLAndPort(srv.Member.Etcd.InitialAdvertisePeerURLs[0])
+		advertisePeerURL, advertisePeerURLPort, err := getURLAndPort(srv.Member.Etcd.AdvertisePeerURLs[0])
 		if err != nil {
 			return err
 		}
@@ -200,7 +211,7 @@ func (srv *Server) stopProxy() {
 	}
 }
 
-func (srv *Server) createEtcdFile() error {
+func (srv *Server) saveEtcdLogFile() error {
 	var err error
 	srv.etcdLogFile, err = os.Create(srv.Member.EtcdLogPath)
 	if err != nil {
@@ -225,6 +236,128 @@ func (srv *Server) creatEtcdCmd() {
 	srv.etcdCmd.Stderr = srv.etcdLogFile
 }
 
+func (srv *Server) saveTLSAssets() error {
+	// if started with manual TLS, stores TLS assets
+	// from tester/client to disk before starting etcd process
+	// TODO: not implemented yet
+	if !srv.Member.Etcd.ClientAutoTLS {
+		if srv.Member.Etcd.ClientCertAuth {
+			return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.ClientCertAuth is %v", srv.Member.Etcd.ClientCertAuth)
+		}
+		if srv.Member.Etcd.ClientCertFile != "" {
+			return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.ClientCertFile is %q", srv.Member.Etcd.ClientCertFile)
+		}
+		if srv.Member.Etcd.ClientKeyFile != "" {
+			return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.ClientKeyFile is %q", srv.Member.Etcd.ClientKeyFile)
+		}
+		if srv.Member.Etcd.ClientTrustedCAFile != "" {
+			return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.ClientTrustedCAFile is %q", srv.Member.Etcd.ClientTrustedCAFile)
+		}
+	}
+	if !srv.Member.Etcd.PeerAutoTLS {
+		if srv.Member.Etcd.PeerClientCertAuth {
+			return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.PeerClientCertAuth is %v", srv.Member.Etcd.PeerClientCertAuth)
+		}
+		if srv.Member.Etcd.PeerCertFile != "" {
+			return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.PeerCertFile is %q", srv.Member.Etcd.PeerCertFile)
+		}
+		if srv.Member.Etcd.PeerKeyFile != "" {
+			return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.PeerKeyFile is %q", srv.Member.Etcd.PeerKeyFile)
+		}
+		if srv.Member.Etcd.PeerTrustedCAFile != "" {
+			return fmt.Errorf("manual TLS setup is not implemented yet, but Member.Etcd.PeerTrustedCAFile is %q", srv.Member.Etcd.PeerTrustedCAFile)
+		}
+	}
+
+	// TODO
+	return nil
+}
+
+func (srv *Server) loadAutoTLSAssets() error {
+	// if started with auto TLS, sends back TLS assets to tester/client
+	if srv.Member.Etcd.ClientAutoTLS {
+		// in case of slow disk
+		time.Sleep(time.Second)
+
+		fdir := filepath.Join(srv.Member.Etcd.DataDir, "fixtures", "client")
+
+		srv.lg.Info(
+			"loading client TLS assets",
+			zap.String("dir", fdir),
+			zap.String("endpoint", srv.EtcdClientEndpoint),
+		)
+
+		certPath := filepath.Join(fdir, "cert.pem")
+		if !fileutil.Exist(certPath) {
+			return fmt.Errorf("cannot find %q", certPath)
+		}
+		certData, err := ioutil.ReadFile(certPath)
+		if err != nil {
+			return fmt.Errorf("cannot read %q (%v)", certPath, err)
+		}
+		srv.Member.ClientCertData = string(certData)
+
+		keyPath := filepath.Join(fdir, "key.pem")
+		if !fileutil.Exist(keyPath) {
+			return fmt.Errorf("cannot find %q", keyPath)
+		}
+		keyData, err := ioutil.ReadFile(keyPath)
+		if err != nil {
+			return fmt.Errorf("cannot read %q (%v)", keyPath, err)
+		}
+		srv.Member.ClientKeyData = string(keyData)
+
+		srv.lg.Info(
+			"loaded client TLS assets",
+			zap.String("peer-cert-path", certPath),
+			zap.Int("peer-cert-length", len(certData)),
+			zap.String("peer-key-path", keyPath),
+			zap.Int("peer-key-length", len(keyData)),
+		)
+	}
+	if srv.Member.Etcd.ClientAutoTLS {
+		// in case of slow disk
+		time.Sleep(time.Second)
+
+		fdir := filepath.Join(srv.Member.Etcd.DataDir, "fixtures", "peer")
+
+		srv.lg.Info(
+			"loading client TLS assets",
+			zap.String("dir", fdir),
+			zap.String("endpoint", srv.EtcdClientEndpoint),
+		)
+
+		certPath := filepath.Join(fdir, "cert.pem")
+		if !fileutil.Exist(certPath) {
+			return fmt.Errorf("cannot find %q", certPath)
+		}
+		certData, err := ioutil.ReadFile(certPath)
+		if err != nil {
+			return fmt.Errorf("cannot read %q (%v)", certPath, err)
+		}
+		srv.Member.PeerCertData = string(certData)
+
+		keyPath := filepath.Join(fdir, "key.pem")
+		if !fileutil.Exist(keyPath) {
+			return fmt.Errorf("cannot find %q", keyPath)
+		}
+		keyData, err := ioutil.ReadFile(keyPath)
+		if err != nil {
+			return fmt.Errorf("cannot read %q (%v)", keyPath, err)
+		}
+		srv.Member.PeerKeyData = string(keyData)
+
+		srv.lg.Info(
+			"loaded peer TLS assets",
+			zap.String("peer-cert-path", certPath),
+			zap.Int("peer-cert-length", len(certData)),
+			zap.String("peer-key-path", keyPath),
+			zap.Int("peer-key-length", len(keyData)),
+		)
+	}
+	return nil
+}
+
 // start but do not wait for it to complete
 func (srv *Server) startEtcdCmd() error {
 	return srv.etcdCmd.Start()
@@ -233,12 +366,17 @@ func (srv *Server) startEtcdCmd() error {
 func (srv *Server) handleRestartEtcd() (*rpcpb.Response, error) {
 	srv.creatEtcdCmd()
 
-	srv.lg.Info("restarting etcd")
-	err := srv.startEtcdCmd()
-	if err != nil {
+	var err error
+	if err = srv.saveTLSAssets(); err != nil {
+		return nil, err
+	}
+	if err = srv.startEtcdCmd(); err != nil {
 		return nil, err
 	}
 	srv.lg.Info("restarted etcd", zap.String("command-path", srv.etcdCmd.Path))
+	if err = srv.loadAutoTLSAssets(); err != nil {
+		return nil, err
+	}
 
 	// wait some time for etcd listener start
 	// before setting up proxy
@@ -251,7 +389,8 @@ func (srv *Server) handleRestartEtcd() (*rpcpb.Response, error) {
 
 	return &rpcpb.Response{
 		Success: true,
-		Status:  "successfully restarted etcd!",
+		Status:  "restart etcd PASS",
+		Member:  srv.Member,
 	}, nil
 }
 
@@ -293,7 +432,7 @@ func (srv *Server) handleFailArchive() (*rpcpb.Response, error) {
 	}
 	srv.lg.Info("archived data", zap.String("base-dir", srv.Member.BaseDir))
 
-	if err = srv.createEtcdFile(); err != nil {
+	if err = srv.saveEtcdLogFile(); err != nil {
 		return nil, err
 	}
 

tools/functional-tester/rpcpb/etcd_config.go

@@ -30,8 +30,19 @@ var etcdFields = []string{
 
 	"ListenClientURLs",
 	"AdvertiseClientURLs",
+	"ClientAutoTLS",
+	"ClientCertAuth",
+	"ClientCertFile",
+	"ClientKeyFile",
+	"ClientTrustedCAFile",
+
 	"ListenPeerURLs",
-	"InitialAdvertisePeerURLs",
+	"AdvertisePeerURLs",
+	"PeerAutoTLS",
+	"PeerClientCertAuth",
+	"PeerCertFile",
+	"PeerKeyFile",
+	"PeerTrustedCAFile",
 
 	"InitialCluster",
 	"InitialClusterState",
@@ -72,12 +83,17 @@ func (cfg *Etcd) Flags() (fs []string) {
 		default:
 			panic(fmt.Errorf("field %q (%v) cannot be parsed", name, fv.Type().Kind()))
 		}
+
 		fname := field.Tag.Get("yaml")
+
 		// TODO: remove this
 		if fname == "initial-corrupt-check" {
 			fname = "experimental-" + fname
 		}
-		fs = append(fs, fmt.Sprintf("--%s=%s", fname, sv))
+
+		if sv != "" {
+			fs = append(fs, fmt.Sprintf("--%s=%s", fname, sv))
+		}
 	}
 	return fs
 }

tools/functional-tester/rpcpb/etcd_config_test.go

@@ -21,34 +21,55 @@ import (
 
 func TestEtcdFlags(t *testing.T) {
 	cfg := &Etcd{
-		Name:                     "s1",
-		DataDir:                  "/tmp/etcd-agent-data-1/etcd.data",
-		WALDir:                   "/tmp/etcd-agent-data-1/etcd.data/member/wal",
-		HeartbeatIntervalMs:      100,
-		ElectionTimeoutMs:        1000,
-		ListenClientURLs:         []string{"127.0.0.1:1379"},
-		AdvertiseClientURLs:      []string{"127.0.0.1:13790"},
-		ListenPeerURLs:           []string{"127.0.0.1:1380"},
-		InitialAdvertisePeerURLs: []string{"127.0.0.1:13800"},
-		InitialCluster:           "s1=127.0.0.1:13800,s2=127.0.0.1:23800,s3=127.0.0.1:33800",
-		InitialClusterState:      "new",
-		InitialClusterToken:      "tkn",
-		SnapshotCount:            10000,
-		QuotaBackendBytes:        10740000000,
-		PreVote:                  true,
-		InitialCorruptCheck:      true,
+		Name:    "s1",
+		DataDir: "/tmp/etcd-agent-data-1/etcd.data",
+		WALDir:  "/tmp/etcd-agent-data-1/etcd.data/member/wal",
+
+		HeartbeatIntervalMs: 100,
+		ElectionTimeoutMs:   1000,
+
+		ListenClientURLs:    []string{"https://127.0.0.1:1379"},
+		AdvertiseClientURLs: []string{"https://127.0.0.1:13790"},
+		ClientAutoTLS:       true,
+		ClientCertAuth:      false,
+		ClientCertFile:      "",
+		ClientKeyFile:       "",
+		ClientTrustedCAFile: "",
+
+		ListenPeerURLs:     []string{"https://127.0.0.1:1380"},
+		AdvertisePeerURLs:  []string{"https://127.0.0.1:13800"},
+		PeerAutoTLS:        true,
+		PeerClientCertAuth: false,
+		PeerCertFile:       "",
+		PeerKeyFile:        "",
+		PeerTrustedCAFile:  "",
+
+		InitialCluster:      "s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800",
+		InitialClusterState: "new",
+		InitialClusterToken: "tkn",
+
+		SnapshotCount:     10000,
+		QuotaBackendBytes: 10740000000,
+
+		PreVote:             true,
+		InitialCorruptCheck: true,
 	}
+
 	exp := []string{
 		"--name=s1",
 		"--data-dir=/tmp/etcd-agent-data-1/etcd.data",
 		"--wal-dir=/tmp/etcd-agent-data-1/etcd.data/member/wal",
 		"--heartbeat-interval=100",
 		"--election-timeout=1000",
-		"--listen-client-urls=127.0.0.1:1379",
-		"--advertise-client-urls=127.0.0.1:13790",
-		"--listen-peer-urls=127.0.0.1:1380",
-		"--initial-advertise-peer-urls=127.0.0.1:13800",
-		"--initial-cluster=s1=127.0.0.1:13800,s2=127.0.0.1:23800,s3=127.0.0.1:33800",
+		"--listen-client-urls=https://127.0.0.1:1379",
+		"--advertise-client-urls=https://127.0.0.1:13790",
+		"--auto-tls=true",
+		"--client-cert-auth=false",
+		"--listen-peer-urls=https://127.0.0.1:1380",
+		"--initial-advertise-peer-urls=https://127.0.0.1:13800",
+		"--peer-auto-tls=true",
+		"--peer-client-cert-auth=false",
+		"--initial-cluster=s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800",
 		"--initial-cluster-state=new",
 		"--initial-cluster-token=tkn",
 		"--snapshot-count=10000",

tools/functional-tester/rpcpb/member.go

@@ -17,39 +17,93 @@ package rpcpb
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"time"
 
 	"github.com/coreos/etcd/clientv3"
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
+	"github.com/coreos/etcd/pkg/transport"
 
 	grpc "google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
 )
 
-var dialOpts = []grpc.DialOption{
-	grpc.WithInsecure(),
-	grpc.WithTimeout(5 * time.Second),
-	grpc.WithBlock(),
-}
-
 // DialEtcdGRPCServer creates a raw gRPC connection to an etcd member.
 func (m *Member) DialEtcdGRPCServer(opts ...grpc.DialOption) (*grpc.ClientConn, error) {
-	if m.EtcdClientTLS {
-		// TODO: support TLS
-		panic("client TLS not supported yet")
-	}
-	return grpc.Dial(m.EtcdClientEndpoint, append(dialOpts, opts...)...)
+	dialOpts := []grpc.DialOption{
+		grpc.WithTimeout(5 * time.Second),
+		grpc.WithBlock(),
+	}
+
+	secure := false
+	for _, cu := range m.Etcd.AdvertiseClientURLs {
+		u, err := url.Parse(cu)
+		if err != nil {
+			return nil, err
+		}
+		if u.Scheme == "https" { // TODO: handle unix
+			secure = true
+		}
+	}
+
+	if secure {
+		// assume save TLS assets are already stord on disk
+		tlsInfo := transport.TLSInfo{
+			CertFile:      m.ClientCertPath,
+			KeyFile:       m.ClientKeyPath,
+			TrustedCAFile: m.ClientTrustedCAPath,
+
+			// TODO: remove this with generated certs
+			// only need it for auto TLS
+			InsecureSkipVerify: true,
+		}
+		tlsConfig, err := tlsInfo.ClientConfig()
+		if err != nil {
+			return nil, err
+		}
+		creds := credentials.NewTLS(tlsConfig)
+		dialOpts = append(dialOpts, grpc.WithTransportCredentials(creds))
+	} else {
+		dialOpts = append(dialOpts, grpc.WithInsecure())
+	}
+	dialOpts = append(dialOpts, opts...)
+	return grpc.Dial(m.EtcdClientEndpoint, dialOpts...)
 }
 
 // CreateEtcdClient creates a client from member.
 func (m *Member) CreateEtcdClient(opts ...grpc.DialOption) (*clientv3.Client, error) {
+	secure := false
+	for _, cu := range m.Etcd.AdvertiseClientURLs {
+		u, err := url.Parse(cu)
+		if err != nil {
+			return nil, err
+		}
+		if u.Scheme == "https" { // TODO: handle unix
+			secure = true
+		}
+	}
+
 	cfg := clientv3.Config{
 		Endpoints:   []string{m.EtcdClientEndpoint},
 		DialTimeout: 5 * time.Second,
 		DialOptions: opts,
 	}
-	if m.EtcdClientTLS {
-		// TODO: support TLS
-		panic("client TLS not supported yet")
+	if secure {
+		// assume save TLS assets are already stord on disk
+		tlsInfo := transport.TLSInfo{
+			CertFile:      m.ClientCertPath,
+			KeyFile:       m.ClientKeyPath,
+			TrustedCAFile: m.ClientTrustedCAPath,
+
+			// TODO: remove this with generated certs
+			// only need it for auto TLS
+			InsecureSkipVerify: true,
+		}
+		tlsConfig, err := tlsInfo.ClientConfig()
+		if err != nil {
+			return nil, err
+		}
+		cfg.TLS = tlsConfig
 	}
 	return clientv3.New(cfg)
 }

tools/functional-tester/rpcpb/rpc.proto

@@ -48,20 +48,29 @@ message Etcd {
 
   repeated string ListenClientURLs = 21 [(gogoproto.moretags) = "yaml:\"listen-client-urls\""];
   repeated string AdvertiseClientURLs = 22 [(gogoproto.moretags) = "yaml:\"advertise-client-urls\""];
-  repeated string ListenPeerURLs = 23 [(gogoproto.moretags) = "yaml:\"listen-peer-urls\""];
-  repeated string InitialAdvertisePeerURLs = 24 [(gogoproto.moretags) = "yaml:\"initial-advertise-peer-urls\""];
-
-  string InitialCluster = 31 [(gogoproto.moretags) = "yaml:\"initial-cluster\""];
-  string InitialClusterState = 32 [(gogoproto.moretags) = "yaml:\"initial-cluster-state\""];
-  string InitialClusterToken = 33 [(gogoproto.moretags) = "yaml:\"initial-cluster-token\""];
-
-  int64 SnapshotCount = 41 [(gogoproto.moretags) = "yaml:\"snapshot-count\""];
-  int64 QuotaBackendBytes = 42 [(gogoproto.moretags) = "yaml:\"quota-backend-bytes\""];
-
-  bool PreVote = 43 [(gogoproto.moretags) = "yaml:\"pre-vote\""];
-  bool InitialCorruptCheck = 44 [(gogoproto.moretags) = "yaml:\"initial-corrupt-check\""];
-
-  // TODO: support TLS
+  bool ClientAutoTLS = 23 [(gogoproto.moretags) = "yaml:\"auto-tls\""];
+  bool ClientCertAuth = 24 [(gogoproto.moretags) = "yaml:\"client-cert-auth\""];
+  string ClientCertFile = 25 [(gogoproto.moretags) = "yaml:\"cert-file\""];
+  string ClientKeyFile = 26 [(gogoproto.moretags) = "yaml:\"key-file\""];
+  string ClientTrustedCAFile = 27 [(gogoproto.moretags) = "yaml:\"trusted-ca-file\""];
+
+  repeated string ListenPeerURLs = 31 [(gogoproto.moretags) = "yaml:\"listen-peer-urls\""];
+  repeated string AdvertisePeerURLs = 32 [(gogoproto.moretags) = "yaml:\"initial-advertise-peer-urls\""];
+  bool PeerAutoTLS = 33 [(gogoproto.moretags) = "yaml:\"peer-auto-tls\""];
+  bool PeerClientCertAuth = 34 [(gogoproto.moretags) = "yaml:\"peer-client-cert-auth\""];
+  string PeerCertFile = 35 [(gogoproto.moretags) = "yaml:\"peer-cert-file\""];
+  string PeerKeyFile = 36 [(gogoproto.moretags) = "yaml:\"peer-key-file\""];
+  string PeerTrustedCAFile = 37 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-file\""];
+
+  string InitialCluster = 41 [(gogoproto.moretags) = "yaml:\"initial-cluster\""];
+  string InitialClusterState = 42 [(gogoproto.moretags) = "yaml:\"initial-cluster-state\""];
+  string InitialClusterToken = 43 [(gogoproto.moretags) = "yaml:\"initial-cluster-token\""];
+
+  int64 SnapshotCount = 51 [(gogoproto.moretags) = "yaml:\"snapshot-count\""];
+  int64 QuotaBackendBytes = 52 [(gogoproto.moretags) = "yaml:\"quota-backend-bytes\""];
+
+  bool PreVote = 63 [(gogoproto.moretags) = "yaml:\"pre-vote\""];
+  bool InitialCorruptCheck = 64 [(gogoproto.moretags) = "yaml:\"initial-corrupt-check\""];
 }
 
 message Member {
@@ -80,19 +89,37 @@ message Member {
   // EtcdLogPath is the log file to store current etcd server logs.
   string EtcdLogPath = 102 [(gogoproto.moretags) = "yaml:\"etcd-log-path\""];
 
-  // EtcdClientTLS is true when client traffic needs to be encrypted.
-  bool EtcdClientTLS = 201 [(gogoproto.moretags) = "yaml:\"etcd-client-tls\""];
   // EtcdClientProxy is true when client traffic needs to be proxied.
   // If true, listen client URL port must be different than advertise client URL port.
-  bool EtcdClientProxy = 202 [(gogoproto.moretags) = "yaml:\"etcd-client-proxy\""];
+  bool EtcdClientProxy = 201 [(gogoproto.moretags) = "yaml:\"etcd-client-proxy\""];
   // EtcdPeerProxy is true when peer traffic needs to be proxied.
   // If true, listen peer URL port must be different than advertise peer URL port.
-  bool EtcdPeerProxy = 203 [(gogoproto.moretags) = "yaml:\"etcd-peer-proxy\""];
-  // EtcdClientEndpoint is the etcd client endpoint.
-  string EtcdClientEndpoint = 204 [(gogoproto.moretags) = "yaml:\"etcd-client-endpoint\""];
+  bool EtcdPeerProxy = 202 [(gogoproto.moretags) = "yaml:\"etcd-peer-proxy\""];
 
+  // EtcdClientEndpoint is the etcd client endpoint.
+  string EtcdClientEndpoint = 301 [(gogoproto.moretags) = "yaml:\"etcd-client-endpoint\""];
   // Etcd defines etcd binary configuration flags.
-  Etcd Etcd = 301 [(gogoproto.moretags) = "yaml:\"etcd\""];
+  Etcd Etcd = 302 [(gogoproto.moretags) = "yaml:\"etcd\""];
+
+  // ClientCertData contains cert file contents from this member's etcd server.
+  string ClientCertData = 401 [(gogoproto.moretags) = "yaml:\"client-cert-data\""];
+  string ClientCertPath = 402 [(gogoproto.moretags) = "yaml:\"client-cert-path\""];
+  // ClientKeyData contains key file contents from this member's etcd server.
+  string ClientKeyData = 403 [(gogoproto.moretags) = "yaml:\"client-key-data\""];
+  string ClientKeyPath = 404 [(gogoproto.moretags) = "yaml:\"client-key-path\""];
+  // ClientTrustedCAData contains trusted CA file contents from this member's etcd server.
+  string ClientTrustedCAData = 405 [(gogoproto.moretags) = "yaml:\"client-trusted-ca-data\""];
+  string ClientTrustedCAPath = 406 [(gogoproto.moretags) = "yaml:\"client-trusted-ca-path\""];
+
+  // PeerCertData contains cert file contents from this member's etcd server.
+  string PeerCertData = 501 [(gogoproto.moretags) = "yaml:\"peer-cert-data\""];
+  string PeerCertPath = 502 [(gogoproto.moretags) = "yaml:\"peer-cert-path\""];
+  // PeerKeyData contains key file contents from this member's etcd server.
+  string PeerKeyData = 503 [(gogoproto.moretags) = "yaml:\"peer-key-data\""];
+  string PeerKeyPath = 504 [(gogoproto.moretags) = "yaml:\"peer-key-path\""];
+  // PeerTrustedCAData contains trusted CA file contents from this member's etcd server.
+  string PeerTrustedCAData = 505 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-data\""];
+  string PeerTrustedCAPath = 506 [(gogoproto.moretags) = "yaml:\"peer-trusted-ca-path\""];
 }
 
 enum FailureCase {
@@ -144,8 +171,9 @@ enum StressType {
 }
 
 message Tester {
-  string TesterNetwork = 1 [(gogoproto.moretags) = "yaml:\"tester-network\""];
-  string TesterAddr = 2 [(gogoproto.moretags) = "yaml:\"tester-addr\""];
+  string TesterDataDir = 1 [(gogoproto.moretags) = "yaml:\"tester-data-dir\""];
+  string TesterNetwork = 2 [(gogoproto.moretags) = "yaml:\"tester-network\""];
+  string TesterAddr = 3 [(gogoproto.moretags) = "yaml:\"tester-addr\""];
 
   // DelayLatencyMsRv is the delay latency in milliseconds,
   // to inject to simulated slow network.
@@ -207,14 +235,15 @@ message Tester {
 
 message Request {
   Operation Operation = 1;
-
+  // Member contains the same Member object from tester configuration.
   Member Member = 2;
+  // Tester contains tester configuration.
   Tester Tester = 3;
 }
 
 message Response {
   bool Success = 1;
   string Status = 2;
-
-  // TODO: support TLS
+  // Member contains the same Member object from tester request.
+  Member Member = 3;
 }

tools/functional-tester/tester/cluster.go

@@ -21,11 +21,13 @@ import (
 	"io/ioutil"
 	"math/rand"
 	"net/http"
+	"net/url"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/coreos/etcd/pkg/debugutil"
+	"github.com/coreos/etcd/pkg/fileutil"
 	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -72,43 +74,43 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
 		return nil, err
 	}
 
-	for i := range clus.Members {
-		if clus.Members[i].BaseDir == "" {
-			return nil, fmt.Errorf("Members[i].BaseDir cannot be empty (got %q)", clus.Members[i].BaseDir)
+	for i, mem := range clus.Members {
+		if mem.BaseDir == "" {
+			return nil, fmt.Errorf("Members[i].BaseDir cannot be empty (got %q)", mem.BaseDir)
 		}
-		if clus.Members[i].EtcdLogPath == "" {
-			return nil, fmt.Errorf("Members[i].EtcdLogPath cannot be empty (got %q)", clus.Members[i].EtcdLogPath)
+		if mem.EtcdLogPath == "" {
+			return nil, fmt.Errorf("Members[i].EtcdLogPath cannot be empty (got %q)", mem.EtcdLogPath)
 		}
 
-		if clus.Members[i].Etcd.Name == "" {
-			return nil, fmt.Errorf("'--name' cannot be empty (got %+v)", clus.Members[i])
+		if mem.Etcd.Name == "" {
+			return nil, fmt.Errorf("'--name' cannot be empty (got %+v)", mem)
 		}
-		if clus.Members[i].Etcd.DataDir == "" {
-			return nil, fmt.Errorf("'--data-dir' cannot be empty (got %+v)", clus.Members[i])
+		if mem.Etcd.DataDir == "" {
+			return nil, fmt.Errorf("'--data-dir' cannot be empty (got %+v)", mem)
 		}
-		if clus.Members[i].Etcd.SnapshotCount == 0 {
-			return nil, fmt.Errorf("'--snapshot-count' cannot be 0 (got %+v)", clus.Members[i].Etcd.SnapshotCount)
+		if mem.Etcd.SnapshotCount == 0 {
+			return nil, fmt.Errorf("'--snapshot-count' cannot be 0 (got %+v)", mem.Etcd.SnapshotCount)
 		}
-		if clus.Members[i].Etcd.DataDir == "" {
-			return nil, fmt.Errorf("'--data-dir' cannot be empty (got %q)", clus.Members[i].Etcd.DataDir)
+		if mem.Etcd.DataDir == "" {
+			return nil, fmt.Errorf("'--data-dir' cannot be empty (got %q)", mem.Etcd.DataDir)
 		}
-		if clus.Members[i].Etcd.WALDir == "" {
-			clus.Members[i].Etcd.WALDir = filepath.Join(clus.Members[i].Etcd.DataDir, "member", "wal")
+		if mem.Etcd.WALDir == "" {
+			clus.Members[i].Etcd.WALDir = filepath.Join(mem.Etcd.DataDir, "member", "wal")
 		}
 
-		if clus.Members[i].Etcd.HeartbeatIntervalMs == 0 {
-			return nil, fmt.Errorf("'--heartbeat-interval' cannot be 0 (got %+v)", clus.Members[i].Etcd)
+		if mem.Etcd.HeartbeatIntervalMs == 0 {
+			return nil, fmt.Errorf("'--heartbeat-interval' cannot be 0 (got %+v)", mem.Etcd)
 		}
-		if clus.Members[i].Etcd.ElectionTimeoutMs == 0 {
-			return nil, fmt.Errorf("'--election-timeout' cannot be 0 (got %+v)", clus.Members[i].Etcd)
+		if mem.Etcd.ElectionTimeoutMs == 0 {
+			return nil, fmt.Errorf("'--election-timeout' cannot be 0 (got %+v)", mem.Etcd)
 		}
-		if int64(clus.Tester.DelayLatencyMs) <= clus.Members[i].Etcd.ElectionTimeoutMs {
-			return nil, fmt.Errorf("delay latency %d ms must be greater than election timeout %d ms", clus.Tester.DelayLatencyMs, clus.Members[i].Etcd.ElectionTimeoutMs)
+		if int64(clus.Tester.DelayLatencyMs) <= mem.Etcd.ElectionTimeoutMs {
+			return nil, fmt.Errorf("delay latency %d ms must be greater than election timeout %d ms", clus.Tester.DelayLatencyMs, mem.Etcd.ElectionTimeoutMs)
 		}
 
 		port := ""
 		listenClientPorts := make([]string, len(clus.Members))
-		for i, u := range clus.Members[i].Etcd.ListenClientURLs {
+		for i, u := range mem.Etcd.ListenClientURLs {
 			if !isValidURL(u) {
 				return nil, fmt.Errorf("'--listen-client-urls' has valid URL %q", u)
 			}
@@ -117,7 +119,7 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
 				return nil, fmt.Errorf("'--listen-client-urls' has no port %q", u)
 			}
 		}
-		for i, u := range clus.Members[i].Etcd.AdvertiseClientURLs {
+		for i, u := range mem.Etcd.AdvertiseClientURLs {
 			if !isValidURL(u) {
 				return nil, fmt.Errorf("'--advertise-client-urls' has valid URL %q", u)
 			}
@@ -125,13 +127,13 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
 			if err != nil {
 				return nil, fmt.Errorf("'--advertise-client-urls' has no port %q", u)
 			}
-			if clus.Members[i].EtcdClientProxy && listenClientPorts[i] == port {
+			if mem.EtcdClientProxy && listenClientPorts[i] == port {
 				return nil, fmt.Errorf("clus.Members[%d] requires client port proxy, but advertise port %q conflicts with listener port %q", i, port, listenClientPorts[i])
 			}
 		}
 
 		listenPeerPorts := make([]string, len(clus.Members))
-		for i, u := range clus.Members[i].Etcd.ListenPeerURLs {
+		for i, u := range mem.Etcd.ListenPeerURLs {
 			if !isValidURL(u) {
 				return nil, fmt.Errorf("'--listen-peer-urls' has valid URL %q", u)
 			}
@@ -140,7 +142,7 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
 				return nil, fmt.Errorf("'--listen-peer-urls' has no port %q", u)
 			}
 		}
-		for i, u := range clus.Members[i].Etcd.InitialAdvertisePeerURLs {
+		for j, u := range mem.Etcd.AdvertisePeerURLs {
 			if !isValidURL(u) {
 				return nil, fmt.Errorf("'--initial-advertise-peer-urls' has valid URL %q", u)
 			}
@@ -148,28 +150,105 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
 			if err != nil {
 				return nil, fmt.Errorf("'--initial-advertise-peer-urls' has no port %q", u)
 			}
-			if clus.Members[i].EtcdPeerProxy && listenPeerPorts[i] == port {
-				return nil, fmt.Errorf("clus.Members[%d] requires peer port proxy, but advertise port %q conflicts with listener port %q", i, port, listenPeerPorts[i])
+			if mem.EtcdPeerProxy && listenPeerPorts[j] == port {
+				return nil, fmt.Errorf("clus.Members[%d] requires peer port proxy, but advertise port %q conflicts with listener port %q", i, port, listenPeerPorts[j])
 			}
 		}
 
-		if !strings.HasPrefix(clus.Members[i].EtcdLogPath, clus.Members[i].BaseDir) {
-			return nil, fmt.Errorf("EtcdLogPath must be prefixed with BaseDir (got %q)", clus.Members[i].EtcdLogPath)
+		if !strings.HasPrefix(mem.EtcdLogPath, mem.BaseDir) {
+			return nil, fmt.Errorf("EtcdLogPath must be prefixed with BaseDir (got %q)", mem.EtcdLogPath)
 		}
-		if !strings.HasPrefix(clus.Members[i].Etcd.DataDir, clus.Members[i].BaseDir) {
-			return nil, fmt.Errorf("Etcd.DataDir must be prefixed with BaseDir (got %q)", clus.Members[i].Etcd.DataDir)
+		if !strings.HasPrefix(mem.Etcd.DataDir, mem.BaseDir) {
+			return nil, fmt.Errorf("Etcd.DataDir must be prefixed with BaseDir (got %q)", mem.Etcd.DataDir)
 		}
 
 		// TODO: support separate WALDir that can be handled via failure-archive
-		if !strings.HasPrefix(clus.Members[i].Etcd.WALDir, clus.Members[i].BaseDir) {
-			return nil, fmt.Errorf("Etcd.WALDir must be prefixed with BaseDir (got %q)", clus.Members[i].Etcd.WALDir)
+		if !strings.HasPrefix(mem.Etcd.WALDir, mem.BaseDir) {
+			return nil, fmt.Errorf("Etcd.WALDir must be prefixed with BaseDir (got %q)", mem.Etcd.WALDir)
 		}
 
-		if len(clus.Tester.FailureCases) == 0 {
-			return nil, errors.New("FailureCases not found")
+		// TODO: only support generated certs with TLS generator
+		// deprecate auto TLS
+		if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertAuth {
+			return nil, fmt.Errorf("Etcd.ClientAutoTLS and Etcd.ClientCertAuth are both 'true'")
+		}
+		if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientCertFile != "" {
+			return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientCertFile is %q", mem.Etcd.ClientCertFile)
+		}
+		if mem.Etcd.ClientCertAuth && mem.Etcd.ClientCertFile == "" {
+			return nil, fmt.Errorf("Etcd.ClientCertAuth 'true', but Etcd.ClientCertFile is %q", mem.Etcd.PeerCertFile)
+		}
+		if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientKeyFile != "" {
+			return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientKeyFile is %q", mem.Etcd.ClientKeyFile)
+		}
+		if mem.Etcd.ClientAutoTLS && mem.Etcd.ClientTrustedCAFile != "" {
+			return nil, fmt.Errorf("Etcd.ClientAutoTLS 'true', but Etcd.ClientTrustedCAFile is %q", mem.Etcd.ClientTrustedCAFile)
+		}
+		if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerClientCertAuth {
+			return nil, fmt.Errorf("Etcd.PeerAutoTLS and Etcd.PeerClientCertAuth are both 'true'")
+		}
+		if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerCertFile != "" {
+			return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile)
+		}
+		if mem.Etcd.PeerClientCertAuth && mem.Etcd.PeerCertFile == "" {
+			return nil, fmt.Errorf("Etcd.PeerClientCertAuth 'true', but Etcd.PeerCertFile is %q", mem.Etcd.PeerCertFile)
+		}
+		if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerKeyFile != "" {
+			return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerKeyFile is %q", mem.Etcd.PeerKeyFile)
+		}
+		if mem.Etcd.PeerAutoTLS && mem.Etcd.PeerTrustedCAFile != "" {
+			return nil, fmt.Errorf("Etcd.PeerAutoTLS 'true', but Etcd.PeerTrustedCAFile is %q", mem.Etcd.PeerTrustedCAFile)
+		}
+
+		if mem.Etcd.ClientAutoTLS || mem.Etcd.ClientCertFile != "" {
+			for _, cu := range mem.Etcd.ListenClientURLs {
+				var u *url.URL
+				u, err = url.Parse(cu)
+				if err != nil {
+					return nil, err
+				}
+				if u.Scheme != "https" { // TODO: support unix
+					return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu)
+				}
+			}
+			for _, cu := range mem.Etcd.AdvertiseClientURLs {
+				var u *url.URL
+				u, err = url.Parse(cu)
+				if err != nil {
+					return nil, err
+				}
+				if u.Scheme != "https" { // TODO: support unix
+					return nil, fmt.Errorf("client TLS is enabled with wrong scheme %q", cu)
+				}
+			}
+		}
+		if mem.Etcd.PeerAutoTLS || mem.Etcd.PeerCertFile != "" {
+			for _, cu := range mem.Etcd.ListenPeerURLs {
+				var u *url.URL
+				u, err = url.Parse(cu)
+				if err != nil {
+					return nil, err
+				}
+				if u.Scheme != "https" { // TODO: support unix
+					return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu)
+				}
+			}
+			for _, cu := range mem.Etcd.AdvertisePeerURLs {
+				var u *url.URL
+				u, err = url.Parse(cu)
+				if err != nil {
+					return nil, err
+				}
+				if u.Scheme != "https" { // TODO: support unix
+					return nil, fmt.Errorf("peer TLS is enabled with wrong scheme %q", cu)
+				}
+			}
 		}
 	}
 
+	if len(clus.Tester.FailureCases) == 0 {
+		return nil, errors.New("FailureCases not found")
+	}
 	if clus.Tester.DelayLatencyMs <= clus.Tester.DelayLatencyMsRv*5 {
 		return nil, fmt.Errorf("delay latency %d ms must be greater than 5x of delay latency random variable %d ms", clus.Tester.DelayLatencyMs, clus.Tester.DelayLatencyMsRv)
 	}
@@ -198,8 +277,6 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
 	return clus, err
 }
 
-// TODO: status handler
-
 var dialOpts = []grpc.DialOption{
 	grpc.WithInsecure(),
 	grpc.WithTimeout(5 * time.Second),
@@ -547,9 +624,79 @@ func (clus *Cluster) sendOperation(idx int, op rpcpb.Operation) error {
 	}
 
 	if !resp.Success {
-		err = errors.New(resp.Status)
+		return errors.New(resp.Status)
 	}
-	return err
+
+	m, secure := clus.Members[idx], false
+	for _, cu := range m.Etcd.AdvertiseClientURLs {
+		u, err := url.Parse(cu)
+		if err != nil {
+			return err
+		}
+		if u.Scheme == "https" { // TODO: handle unix
+			secure = true
+		}
+	}
+
+	// store TLS assets from agents/servers onto disk
+	if secure && (op == rpcpb.Operation_InitialStartEtcd || op == rpcpb.Operation_RestartEtcd) {
+		dirClient := filepath.Join(
+			clus.Tester.TesterDataDir,
+			clus.Members[idx].Etcd.Name,
+			"fixtures",
+			"client",
+		)
+		if err = fileutil.TouchDirAll(dirClient); err != nil {
+			return err
+		}
+
+		clientCertData := []byte(resp.Member.ClientCertData)
+		if len(clientCertData) == 0 {
+			return fmt.Errorf("got empty client cert from %q", m.EtcdClientEndpoint)
+		}
+		clientCertPath := filepath.Join(dirClient, "cert.pem")
+		if err = ioutil.WriteFile(clientCertPath, clientCertData, 0644); err != nil { // overwrite if exists
+			return err
+		}
+		resp.Member.ClientCertPath = clientCertPath
+		clus.lg.Info(
+			"saved client cert file",
+			zap.String("path", clientCertPath),
+		)
+
+		clientKeyData := []byte(resp.Member.ClientKeyData)
+		if len(clientKeyData) == 0 {
+			return fmt.Errorf("got empty client key from %q", m.EtcdClientEndpoint)
+		}
+		clientKeyPath := filepath.Join(dirClient, "key.pem")
+		if err = ioutil.WriteFile(clientKeyPath, clientKeyData, 0644); err != nil { // overwrite if exists
+			return err
+		}
+		resp.Member.ClientKeyPath = clientKeyPath
+		clus.lg.Info(
+			"saved client key file",
+			zap.String("path", clientKeyPath),
+		)
+
+		clientTrustedCAData := []byte(resp.Member.ClientTrustedCAData)
+		if len(clientTrustedCAData) != 0 {
+			// TODO: disable this when auto TLS is deprecated
+			clientTrustedCAPath := filepath.Join(dirClient, "ca.pem")
+			if err = ioutil.WriteFile(clientTrustedCAPath, clientTrustedCAData, 0644); err != nil { // overwrite if exists
+				return err
+			}
+			resp.Member.ClientTrustedCAPath = clientTrustedCAPath
+			clus.lg.Info(
+				"saved client trusted CA file",
+				zap.String("path", clientTrustedCAPath),
+			)
+		}
+
+		// no need to store peer certs for tester clients
+
+		clus.Members[idx] = resp.Member
+	}
+	return nil
 }
 
 // DestroyEtcdAgents terminates all tester connections to agents and etcd servers.

tools/functional-tester/tester/cluster_test.go

@@ -33,27 +33,36 @@ func Test_newCluster(t *testing.T) {
 				FailpointHTTPAddr:  "http://127.0.0.1:7381",
 				BaseDir:            "/tmp/etcd-agent-data-1",
 				EtcdLogPath:        "/tmp/etcd-agent-data-1/current-etcd.log",
-				EtcdClientTLS:      false,
 				EtcdClientProxy:    false,
 				EtcdPeerProxy:      true,
 				EtcdClientEndpoint: "127.0.0.1:1379",
 				Etcd: &rpcpb.Etcd{
-					Name:                     "s1",
-					DataDir:                  "/tmp/etcd-agent-data-1/etcd.data",
-					WALDir:                   "/tmp/etcd-agent-data-1/etcd.data/member/wal",
-					HeartbeatIntervalMs:      100,
-					ElectionTimeoutMs:        1000,
-					ListenClientURLs:         []string{"http://127.0.0.1:1379"},
-					AdvertiseClientURLs:      []string{"http://127.0.0.1:1379"},
-					ListenPeerURLs:           []string{"http://127.0.0.1:1380"},
-					InitialAdvertisePeerURLs: []string{"http://127.0.0.1:13800"},
-					InitialCluster:           "s1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800",
-					InitialClusterState:      "new",
-					InitialClusterToken:      "tkn",
-					SnapshotCount:            10000,
-					QuotaBackendBytes:        10740000000,
-					PreVote:                  true,
-					InitialCorruptCheck:      true,
+					Name:                "s1",
+					DataDir:             "/tmp/etcd-agent-data-1/etcd.data",
+					WALDir:              "/tmp/etcd-agent-data-1/etcd.data/member/wal",
+					HeartbeatIntervalMs: 100,
+					ElectionTimeoutMs:   1000,
+					ListenClientURLs:    []string{"https://127.0.0.1:1379"},
+					AdvertiseClientURLs: []string{"https://127.0.0.1:1379"},
+					ClientAutoTLS:       true,
+					ClientCertAuth:      false,
+					ClientCertFile:      "",
+					ClientKeyFile:       "",
+					ClientTrustedCAFile: "",
+					ListenPeerURLs:      []string{"https://127.0.0.1:1380"},
+					AdvertisePeerURLs:   []string{"https://127.0.0.1:13800"},
+					PeerAutoTLS:         true,
+					PeerClientCertAuth:  false,
+					PeerCertFile:        "",
+					PeerKeyFile:         "",
+					PeerTrustedCAFile:   "",
+					InitialCluster:      "s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800",
+					InitialClusterState: "new",
+					InitialClusterToken: "tkn",
+					SnapshotCount:       10000,
+					QuotaBackendBytes:   10740000000,
+					PreVote:             true,
+					InitialCorruptCheck: true,
 				},
 			},
 			{
@@ -62,27 +71,36 @@ func Test_newCluster(t *testing.T) {
 				FailpointHTTPAddr:  "http://127.0.0.1:7382",
 				BaseDir:            "/tmp/etcd-agent-data-2",
 				EtcdLogPath:        "/tmp/etcd-agent-data-2/current-etcd.log",
-				EtcdClientTLS:      false,
 				EtcdClientProxy:    false,
 				EtcdPeerProxy:      true,
 				EtcdClientEndpoint: "127.0.0.1:2379",
 				Etcd: &rpcpb.Etcd{
-					Name:                     "s2",
-					DataDir:                  "/tmp/etcd-agent-data-2/etcd.data",
-					WALDir:                   "/tmp/etcd-agent-data-2/etcd.data/member/wal",
-					HeartbeatIntervalMs:      100,
-					ElectionTimeoutMs:        1000,
-					ListenClientURLs:         []string{"http://127.0.0.1:2379"},
-					AdvertiseClientURLs:      []string{"http://127.0.0.1:2379"},
-					ListenPeerURLs:           []string{"http://127.0.0.1:2380"},
-					InitialAdvertisePeerURLs: []string{"http://127.0.0.1:23800"},
-					InitialCluster:           "s1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800",
-					InitialClusterState:      "new",
-					InitialClusterToken:      "tkn",
-					SnapshotCount:            10000,
-					QuotaBackendBytes:        10740000000,
-					PreVote:                  true,
-					InitialCorruptCheck:      true,
+					Name:                "s2",
+					DataDir:             "/tmp/etcd-agent-data-2/etcd.data",
+					WALDir:              "/tmp/etcd-agent-data-2/etcd.data/member/wal",
+					HeartbeatIntervalMs: 100,
+					ElectionTimeoutMs:   1000,
+					ListenClientURLs:    []string{"https://127.0.0.1:2379"},
+					AdvertiseClientURLs: []string{"https://127.0.0.1:2379"},
+					ClientAutoTLS:       true,
+					ClientCertAuth:      false,
+					ClientCertFile:      "",
+					ClientKeyFile:       "",
+					ClientTrustedCAFile: "",
+					ListenPeerURLs:      []string{"https://127.0.0.1:2380"},
+					AdvertisePeerURLs:   []string{"https://127.0.0.1:23800"},
+					PeerAutoTLS:         true,
+					PeerClientCertAuth:  false,
+					PeerCertFile:        "",
+					PeerKeyFile:         "",
+					PeerTrustedCAFile:   "",
+					InitialCluster:      "s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800",
+					InitialClusterState: "new",
+					InitialClusterToken: "tkn",
+					SnapshotCount:       10000,
+					QuotaBackendBytes:   10740000000,
+					PreVote:             true,
+					InitialCorruptCheck: true,
 				},
 			},
 			{
@@ -91,31 +109,41 @@ func Test_newCluster(t *testing.T) {
 				FailpointHTTPAddr:  "http://127.0.0.1:7383",
 				BaseDir:            "/tmp/etcd-agent-data-3",
 				EtcdLogPath:        "/tmp/etcd-agent-data-3/current-etcd.log",
-				EtcdClientTLS:      false,
 				EtcdClientProxy:    false,
 				EtcdPeerProxy:      true,
 				EtcdClientEndpoint: "127.0.0.1:3379",
 				Etcd: &rpcpb.Etcd{
-					Name:                     "s3",
-					DataDir:                  "/tmp/etcd-agent-data-3/etcd.data",
-					WALDir:                   "/tmp/etcd-agent-data-3/etcd.data/member/wal",
-					HeartbeatIntervalMs:      100,
-					ElectionTimeoutMs:        1000,
-					ListenClientURLs:         []string{"http://127.0.0.1:3379"},
-					AdvertiseClientURLs:      []string{"http://127.0.0.1:3379"},
-					ListenPeerURLs:           []string{"http://127.0.0.1:3380"},
-					InitialAdvertisePeerURLs: []string{"http://127.0.0.1:33800"},
-					InitialCluster:           "s1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800",
-					InitialClusterState:      "new",
-					InitialClusterToken:      "tkn",
-					SnapshotCount:            10000,
-					QuotaBackendBytes:        10740000000,
-					PreVote:                  true,
-					InitialCorruptCheck:      true,
+					Name:                "s3",
+					DataDir:             "/tmp/etcd-agent-data-3/etcd.data",
+					WALDir:              "/tmp/etcd-agent-data-3/etcd.data/member/wal",
+					HeartbeatIntervalMs: 100,
+					ElectionTimeoutMs:   1000,
+					ListenClientURLs:    []string{"https://127.0.0.1:3379"},
+					AdvertiseClientURLs: []string{"https://127.0.0.1:3379"},
+					ClientAutoTLS:       true,
+					ClientCertAuth:      false,
+					ClientCertFile:      "",
+					ClientKeyFile:       "",
+					ClientTrustedCAFile: "",
+					ListenPeerURLs:      []string{"https://127.0.0.1:3380"},
+					AdvertisePeerURLs:   []string{"https://127.0.0.1:33800"},
+					PeerAutoTLS:         true,
+					PeerClientCertAuth:  false,
+					PeerCertFile:        "",
+					PeerKeyFile:         "",
+					PeerTrustedCAFile:   "",
+					InitialCluster:      "s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800",
+					InitialClusterState: "new",
+					InitialClusterToken: "tkn",
+					SnapshotCount:       10000,
+					QuotaBackendBytes:   10740000000,
+					PreVote:             true,
+					InitialCorruptCheck: true,
 				},
 			},
 		},
 		Tester: &rpcpb.Tester{
+			TesterDataDir:         "/tmp/etcd-tester-data",
 			TesterNetwork:         "tcp",
 			TesterAddr:            "127.0.0.1:9028",
 			DelayLatencyMs:        5000,

tools/functional-tester/tester/cluster_tester.go

@@ -19,6 +19,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/coreos/etcd/pkg/fileutil"
 	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
 
 	"go.uber.org/zap"
@@ -30,7 +31,13 @@ const compactQPS = 50000
 
 // StartTester starts tester.
 func (clus *Cluster) StartTester() {
-	// TODO: upate status
+	if err := fileutil.TouchDirAll(clus.Tester.TesterDataDir); err != nil {
+		clus.lg.Panic(
+			"failed to create test data directory",
+			zap.String("dir", clus.Tester.TesterDataDir),
+			zap.Error(err),
+		)
+	}
 
 	var preModifiedKey int64
 	for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ {
@@ -124,6 +131,7 @@ func (clus *Cluster) doRound() error {
 			zap.Int("round", clus.rd),
 			zap.Int("case", clus.cs),
 			zap.String("desc", fa.Desc()),
+			zap.Int("total-failures", len(clus.failures)),
 		)
 
 		clus.lg.Info("wait health before injecting failures")
@@ -208,6 +216,7 @@ func (clus *Cluster) doRound() error {
 			zap.Int("round", clus.rd),
 			zap.Int("case", clus.cs),
 			zap.String("desc", fa.Desc()),
+			zap.Int("total-failures", len(clus.failures)),
 			zap.Duration("took", time.Since(caseNow)),
 		)
 	}
@@ -216,6 +225,7 @@ func (clus *Cluster) doRound() error {
 		"round ALL PASS",
 		zap.Int("round", clus.rd),
 		zap.Strings("failures", clus.failureStrings()),
+		zap.Int("total-failures", len(clus.failures)),
 		zap.Duration("took", time.Since(roundNow)),
 	)
 	return nil

tools/functional-tester/tester/local-test.yaml

@@ -4,7 +4,6 @@ agent-configs:
   failpoint-http-addr: http://127.0.0.1:7381
   base-dir: /tmp/etcd-agent-data-1
   etcd-log-path: /tmp/etcd-agent-data-1/current-etcd.log
-  etcd-client-tls: false
   etcd-client-proxy: false
   etcd-peer-proxy: true
   etcd-client-endpoint: 127.0.0.1:1379
@@ -14,11 +13,21 @@ agent-configs:
     wal-dir: /tmp/etcd-agent-data-1/etcd.data/member/wal
     heartbeat-interval: 100
     election-timeout: 1000
-    listen-client-urls: ["http://127.0.0.1:1379"]
-    advertise-client-urls: ["http://127.0.0.1:1379"]
-    listen-peer-urls: ["http://127.0.0.1:1380"]
-    initial-advertise-peer-urls: ["http://127.0.0.1:13800"]
-    initial-cluster: s1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800
+    listen-client-urls: ["https://127.0.0.1:1379"]
+    advertise-client-urls: ["https://127.0.0.1:1379"]
+    auto-tls: true
+    client-cert-auth: false
+    cert-file: ""
+    key-file: ""
+    trusted-ca-file: ""
+    listen-peer-urls: ["https://127.0.0.1:1380"]
+    initial-advertise-peer-urls: ["https://127.0.0.1:13800"]
+    peer-auto-tls: true
+    peer-client-cert-auth: false
+    peer-cert-file: ""
+    peer-key-file: ""
+    peer-trusted-ca-file: ""
+    initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800
     initial-cluster-state: new
     initial-cluster-token: tkn
     snapshot-count: 10000
@@ -30,7 +39,6 @@ agent-configs:
   failpoint-http-addr: http://127.0.0.1:7382
   base-dir: /tmp/etcd-agent-data-2
   etcd-log-path: /tmp/etcd-agent-data-2/current-etcd.log
-  etcd-client-tls: false
   etcd-client-proxy: false
   etcd-peer-proxy: true
   etcd-client-endpoint: 127.0.0.1:2379
@@ -40,11 +48,21 @@ agent-configs:
     wal-dir: /tmp/etcd-agent-data-2/etcd.data/member/wal
     heartbeat-interval: 100
     election-timeout: 1000
-    listen-client-urls: ["http://127.0.0.1:2379"]
-    advertise-client-urls: ["http://127.0.0.1:2379"]
-    listen-peer-urls: ["http://127.0.0.1:2380"]
-    initial-advertise-peer-urls: ["http://127.0.0.1:23800"]
-    initial-cluster: s1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800
+    listen-client-urls: ["https://127.0.0.1:2379"]
+    advertise-client-urls: ["https://127.0.0.1:2379"]
+    auto-tls: true
+    client-cert-auth: false
+    cert-file: ""
+    key-file: ""
+    trusted-ca-file: ""
+    listen-peer-urls: ["https://127.0.0.1:2380"]
+    initial-advertise-peer-urls: ["https://127.0.0.1:23800"]
+    peer-auto-tls: true
+    peer-client-cert-auth: false
+    peer-cert-file: ""
+    peer-key-file: ""
+    peer-trusted-ca-file: ""
+    initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800
     initial-cluster-state: new
     initial-cluster-token: tkn
     snapshot-count: 10000
@@ -56,7 +74,6 @@ agent-configs:
   failpoint-http-addr: http://127.0.0.1:7383
   base-dir: /tmp/etcd-agent-data-3
   etcd-log-path: /tmp/etcd-agent-data-3/current-etcd.log
-  etcd-client-tls: false
   etcd-client-proxy: false
   etcd-peer-proxy: true
   etcd-client-endpoint: 127.0.0.1:3379
@@ -66,11 +83,21 @@ agent-configs:
     wal-dir: /tmp/etcd-agent-data-3/etcd.data/member/wal
     heartbeat-interval: 100
     election-timeout: 1000
-    listen-client-urls: ["http://127.0.0.1:3379"]
-    advertise-client-urls: ["http://127.0.0.1:3379"]
-    listen-peer-urls: ["http://127.0.0.1:3380"]
-    initial-advertise-peer-urls: ["http://127.0.0.1:33800"]
-    initial-cluster: s1=http://127.0.0.1:13800,s2=http://127.0.0.1:23800,s3=http://127.0.0.1:33800
+    listen-client-urls: ["https://127.0.0.1:3379"]
+    advertise-client-urls: ["https://127.0.0.1:3379"]
+    auto-tls: true
+    client-cert-auth: false
+    cert-file: ""
+    key-file: ""
+    trusted-ca-file: ""
+    listen-peer-urls: ["https://127.0.0.1:3380"]
+    initial-advertise-peer-urls: ["https://127.0.0.1:33800"]
+    peer-auto-tls: true
+    peer-client-cert-auth: false
+    peer-cert-file: ""
+    peer-key-file: ""
+    peer-trusted-ca-file: ""
+    initial-cluster: s1=https://127.0.0.1:13800,s2=https://127.0.0.1:23800,s3=https://127.0.0.1:33800
     initial-cluster-state: new
     initial-cluster-token: tkn
     snapshot-count: 10000
@@ -79,6 +106,7 @@ agent-configs:
     initial-corrupt-check: true
 
 tester-config:
+  tester-data-dir: /tmp/etcd-tester-data
   tester-network: tcp
   tester-addr: 127.0.0.1:9028