functional-tester/tester: send TLS client requests, cleanup

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

tools/functional-tester/tester/cluster.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/coreos/etcd/pkg/debugutil"
+	"github.com/coreos/etcd/pkg/fileutil"
 	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
 
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -201,7 +202,8 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
 
 		if mem.Etcd.ClientAutoTLS || mem.Etcd.ClientCertFile != "" {
 			for _, cu := range mem.Etcd.ListenClientURLs {
-				u, err := url.Parse(cu)
+				var u *url.URL
+				u, err = url.Parse(cu)
 				if err != nil {
 					return nil, err
 				}
@@ -210,7 +212,8 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
 				}
 			}
 			for _, cu := range mem.Etcd.AdvertiseClientURLs {
-				u, err := url.Parse(cu)
+				var u *url.URL
+				u, err = url.Parse(cu)
 				if err != nil {
 					return nil, err
 				}
@@ -221,7 +224,8 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
 		}
 		if mem.Etcd.PeerAutoTLS || mem.Etcd.PeerCertFile != "" {
 			for _, cu := range mem.Etcd.ListenPeerURLs {
-				u, err := url.Parse(cu)
+				var u *url.URL
+				u, err = url.Parse(cu)
 				if err != nil {
 					return nil, err
 				}
@@ -230,7 +234,8 @@ func newCluster(lg *zap.Logger, fpath string) (*Cluster, error) {
 				}
 			}
 			for _, cu := range mem.Etcd.AdvertisePeerURLs {
-				u, err := url.Parse(cu)
+				var u *url.URL
+				u, err = url.Parse(cu)
 				if err != nil {
 					return nil, err
 				}
@@ -619,9 +624,79 @@ func (clus *Cluster) sendOperation(idx int, op rpcpb.Operation) error {
 	}
 
 	if !resp.Success {
-		err = errors.New(resp.Status)
+		return errors.New(resp.Status)
 	}
-	return err
+
+	m, secure := clus.Members[idx], false
+	for _, cu := range m.Etcd.AdvertiseClientURLs {
+		u, err := url.Parse(cu)
+		if err != nil {
+			return err
+		}
+		if u.Scheme == "https" { // TODO: handle unix
+			secure = true
+		}
+	}
+
+	// store TLS assets from agents/servers onto disk
+	if secure && (op == rpcpb.Operation_InitialStartEtcd || op == rpcpb.Operation_RestartEtcd) {
+		dirClient := filepath.Join(
+			clus.Tester.TesterDataDir,
+			clus.Members[idx].Etcd.Name,
+			"fixtures",
+			"client",
+		)
+		if err = fileutil.TouchDirAll(dirClient); err != nil {
+			return err
+		}
+
+		clientCertData := []byte(resp.Member.ClientCertData)
+		if len(clientCertData) == 0 {
+			return fmt.Errorf("got empty client cert from %q", m.EtcdClientEndpoint)
+		}
+		clientCertPath := filepath.Join(dirClient, "cert.pem")
+		if err = ioutil.WriteFile(clientCertPath, clientCertData, 0644); err != nil { // overwrite if exists
+			return err
+		}
+		resp.Member.ClientCertPath = clientCertPath
+		clus.lg.Info(
+			"saved client cert file",
+			zap.String("path", clientCertPath),
+		)
+
+		clientKeyData := []byte(resp.Member.ClientKeyData)
+		if len(clientKeyData) == 0 {
+			return fmt.Errorf("got empty client key from %q", m.EtcdClientEndpoint)
+		}
+		clientKeyPath := filepath.Join(dirClient, "key.pem")
+		if err = ioutil.WriteFile(clientKeyPath, clientKeyData, 0644); err != nil { // overwrite if exists
+			return err
+		}
+		resp.Member.ClientKeyPath = clientKeyPath
+		clus.lg.Info(
+			"saved client key file",
+			zap.String("path", clientKeyPath),
+		)
+
+		clientTrustedCAData := []byte(resp.Member.ClientTrustedCAData)
+		if len(clientTrustedCAData) != 0 {
+			// TODO: disable this when auto TLS is deprecated
+			clientTrustedCAPath := filepath.Join(dirClient, "ca.pem")
+			if err = ioutil.WriteFile(clientTrustedCAPath, clientTrustedCAData, 0644); err != nil { // overwrite if exists
+				return err
+			}
+			resp.Member.ClientTrustedCAPath = clientTrustedCAPath
+			clus.lg.Info(
+				"saved client trusted CA file",
+				zap.String("path", clientTrustedCAPath),
+			)
+		}
+
+		// no need to store peer certs for tester clients
+
+		clus.Members[idx] = resp.Member
+	}
+	return nil
 }
 
 // DestroyEtcdAgents terminates all tester connections to agents and etcd servers.

tools/functional-tester/tester/cluster_test.go

@@ -33,7 +33,6 @@ func Test_newCluster(t *testing.T) {
 				FailpointHTTPAddr:  "http://127.0.0.1:7381",
 				BaseDir:            "/tmp/etcd-agent-data-1",
 				EtcdLogPath:        "/tmp/etcd-agent-data-1/current-etcd.log",
-				EtcdClientTLS:      false,
 				EtcdClientProxy:    false,
 				EtcdPeerProxy:      true,
 				EtcdClientEndpoint: "127.0.0.1:1379",
@@ -72,7 +71,6 @@ func Test_newCluster(t *testing.T) {
 				FailpointHTTPAddr:  "http://127.0.0.1:7382",
 				BaseDir:            "/tmp/etcd-agent-data-2",
 				EtcdLogPath:        "/tmp/etcd-agent-data-2/current-etcd.log",
-				EtcdClientTLS:      false,
 				EtcdClientProxy:    false,
 				EtcdPeerProxy:      true,
 				EtcdClientEndpoint: "127.0.0.1:2379",
@@ -111,7 +109,6 @@ func Test_newCluster(t *testing.T) {
 				FailpointHTTPAddr:  "http://127.0.0.1:7383",
 				BaseDir:            "/tmp/etcd-agent-data-3",
 				EtcdLogPath:        "/tmp/etcd-agent-data-3/current-etcd.log",
-				EtcdClientTLS:      false,
 				EtcdClientProxy:    false,
 				EtcdPeerProxy:      true,
 				EtcdClientEndpoint: "127.0.0.1:3379",
@@ -146,6 +143,7 @@ func Test_newCluster(t *testing.T) {
 			},
 		},
 		Tester: &rpcpb.Tester{
+			TesterDataDir:         "/tmp/etcd-tester-data",
 			TesterNetwork:         "tcp",
 			TesterAddr:            "127.0.0.1:9028",
 			DelayLatencyMs:        5000,

tools/functional-tester/tester/cluster_tester.go

@@ -19,6 +19,7 @@ import (
 	"os"
 	"time"
 
+	"github.com/coreos/etcd/pkg/fileutil"
 	"github.com/coreos/etcd/tools/functional-tester/rpcpb"
 
 	"go.uber.org/zap"
@@ -30,7 +31,13 @@ const compactQPS = 50000
 
 // StartTester starts tester.
 func (clus *Cluster) StartTester() {
-	// TODO: upate status
+	if err := fileutil.TouchDirAll(clus.Tester.TesterDataDir); err != nil {
+		clus.lg.Panic(
+			"failed to create test data directory",
+			zap.String("dir", clus.Tester.TesterDataDir),
+			zap.Error(err),
+		)
+	}
 
 	var preModifiedKey int64
 	for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ {
@@ -124,6 +131,7 @@ func (clus *Cluster) doRound() error {
 			zap.Int("round", clus.rd),
 			zap.Int("case", clus.cs),
 			zap.String("desc", fa.Desc()),
+			zap.Int("total-failures", len(clus.failures)),
 		)
 
 		clus.lg.Info("wait health before injecting failures")
@@ -208,6 +216,7 @@ func (clus *Cluster) doRound() error {
 			zap.Int("round", clus.rd),
 			zap.Int("case", clus.cs),
 			zap.String("desc", fa.Desc()),
+			zap.Int("total-failures", len(clus.failures)),
 			zap.Duration("took", time.Since(caseNow)),
 		)
 	}
@@ -216,6 +225,7 @@ func (clus *Cluster) doRound() error {
 		"round ALL PASS",
 		zap.Int("round", clus.rd),
 		zap.Strings("failures", clus.failureStrings()),
+		zap.Int("total-failures", len(clus.failures)),
 		zap.Duration("took", time.Since(roundNow)),
 	)
 	return nil

tools/functional-tester/tester/local-test.yaml

@@ -4,7 +4,6 @@ agent-configs:
   failpoint-http-addr: http://127.0.0.1:7381
   base-dir: /tmp/etcd-agent-data-1
   etcd-log-path: /tmp/etcd-agent-data-1/current-etcd.log
-  etcd-client-tls: false
   etcd-client-proxy: false
   etcd-peer-proxy: true
   etcd-client-endpoint: 127.0.0.1:1379
@@ -40,7 +39,6 @@ agent-configs:
   failpoint-http-addr: http://127.0.0.1:7382
   base-dir: /tmp/etcd-agent-data-2
   etcd-log-path: /tmp/etcd-agent-data-2/current-etcd.log
-  etcd-client-tls: false
   etcd-client-proxy: false
   etcd-peer-proxy: true
   etcd-client-endpoint: 127.0.0.1:2379
@@ -76,7 +74,6 @@ agent-configs:
   failpoint-http-addr: http://127.0.0.1:7383
   base-dir: /tmp/etcd-agent-data-3
   etcd-log-path: /tmp/etcd-agent-data-3/current-etcd.log
-  etcd-client-tls: false
   etcd-client-proxy: false
   etcd-peer-proxy: true
   etcd-client-endpoint: 127.0.0.1:3379
@@ -109,6 +106,7 @@ agent-configs:
     initial-corrupt-check: true
 
 tester-config:
+  tester-data-dir: /tmp/etcd-tester-data
   tester-network: tcp
   tester-addr: 127.0.0.1:9028