tests/e2e: fix

Signed-off-by: Gyuho Lee <leegyuho@amazon.com>

tests/e2e/cluster_direct_test.go

@@ -0,0 +1,21 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !cluster_proxy
+
+package e2e
+
+func newEtcdProcess(cfg *etcdServerProcessConfig) (etcdProcess, error) {
+	return newEtcdServerProcess(cfg)
+}

tests/e2e/cluster_test.go

@@ -0,0 +1,402 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"strings"
+)
+
+const etcdProcessBasePort = 20000
+
+type clientConnType int
+
+const (
+	clientNonTLS clientConnType = iota
+	clientTLS
+	clientTLSAndNonTLS
+)
+
+var (
+	configNoTLS = etcdProcessClusterConfig{
+		clusterSize:  3,
+		initialToken: "new",
+	}
+	configAutoTLS = etcdProcessClusterConfig{
+		clusterSize:   3,
+		isPeerTLS:     true,
+		isPeerAutoTLS: true,
+		initialToken:  "new",
+	}
+	configTLS = etcdProcessClusterConfig{
+		clusterSize:  3,
+		clientTLS:    clientTLS,
+		isPeerTLS:    true,
+		initialToken: "new",
+	}
+	configClientTLS = etcdProcessClusterConfig{
+		clusterSize:  3,
+		clientTLS:    clientTLS,
+		initialToken: "new",
+	}
+	configClientBoth = etcdProcessClusterConfig{
+		clusterSize:  1,
+		clientTLS:    clientTLSAndNonTLS,
+		initialToken: "new",
+	}
+	configClientAutoTLS = etcdProcessClusterConfig{
+		clusterSize:     1,
+		isClientAutoTLS: true,
+		clientTLS:       clientTLS,
+		initialToken:    "new",
+	}
+	configPeerTLS = etcdProcessClusterConfig{
+		clusterSize:  3,
+		isPeerTLS:    true,
+		initialToken: "new",
+	}
+	configClientTLSCertAuth = etcdProcessClusterConfig{
+		clusterSize:           1,
+		clientTLS:             clientTLS,
+		initialToken:          "new",
+		clientCertAuthEnabled: true,
+	}
+	configClientTLSCertAuthWithNoCN = etcdProcessClusterConfig{
+		clusterSize:           1,
+		clientTLS:             clientTLS,
+		initialToken:          "new",
+		clientCertAuthEnabled: true,
+		noCN:                  true,
+	}
+	configJWT = etcdProcessClusterConfig{
+		clusterSize:   1,
+		initialToken:  "new",
+		authTokenOpts: "jwt,pub-key=../../integration/fixtures/server.crt,priv-key=../../integration/fixtures/server.key.insecure,sign-method=RS256,ttl=1s",
+	}
+)
+
+func configStandalone(cfg etcdProcessClusterConfig) *etcdProcessClusterConfig {
+	ret := cfg
+	ret.clusterSize = 1
+	return &ret
+}
+
+type etcdProcessCluster struct {
+	cfg   *etcdProcessClusterConfig
+	procs []etcdProcess
+}
+
+type etcdProcessClusterConfig struct {
+	execPath    string
+	dataDirPath string
+	keepDataDir bool
+
+	clusterSize int
+
+	baseScheme string
+	basePort   int
+
+	metricsURLScheme string
+
+	snapshotCount int // default is 10000
+
+	clientTLS             clientConnType
+	clientCertAuthEnabled bool
+	isPeerTLS             bool
+	isPeerAutoTLS         bool
+	isClientAutoTLS       bool
+	isClientCRL           bool
+	noCN                  bool
+
+	cipherSuites []string
+
+	forceNewCluster     bool
+	initialToken        string
+	quotaBackendBytes   int64
+	noStrictReconfig    bool
+	enableV2            bool
+	initialCorruptCheck bool
+	authTokenOpts       string
+}
+
+// newEtcdProcessCluster launches a new cluster from etcd processes, returning
+// a new etcdProcessCluster once all nodes are ready to accept client requests.
+func newEtcdProcessCluster(cfg *etcdProcessClusterConfig) (*etcdProcessCluster, error) {
+	etcdCfgs := cfg.etcdServerProcessConfigs()
+	epc := &etcdProcessCluster{
+		cfg:   cfg,
+		procs: make([]etcdProcess, cfg.clusterSize),
+	}
+
+	// launch etcd processes
+	for i := range etcdCfgs {
+		proc, err := newEtcdProcess(etcdCfgs[i])
+		if err != nil {
+			epc.Close()
+			return nil, err
+		}
+		epc.procs[i] = proc
+	}
+
+	if err := epc.Start(); err != nil {
+		return nil, err
+	}
+	return epc, nil
+}
+
+func (cfg *etcdProcessClusterConfig) clientScheme() string {
+	if cfg.clientTLS == clientTLS {
+		return "https"
+	}
+	return "http"
+}
+
+func (cfg *etcdProcessClusterConfig) peerScheme() string {
+	peerScheme := cfg.baseScheme
+	if peerScheme == "" {
+		peerScheme = "http"
+	}
+	if cfg.isPeerTLS {
+		peerScheme += "s"
+	}
+	return peerScheme
+}
+
+func (cfg *etcdProcessClusterConfig) etcdServerProcessConfigs() []*etcdServerProcessConfig {
+	if cfg.basePort == 0 {
+		cfg.basePort = etcdProcessBasePort
+	}
+	if cfg.execPath == "" {
+		cfg.execPath = binPath
+	}
+	if cfg.snapshotCount == 0 {
+		cfg.snapshotCount = 10000
+	}
+
+	etcdCfgs := make([]*etcdServerProcessConfig, cfg.clusterSize)
+	initialCluster := make([]string, cfg.clusterSize)
+	for i := 0; i < cfg.clusterSize; i++ {
+		var curls []string
+		var curl, curltls string
+		port := cfg.basePort + 5*i
+		curlHost := fmt.Sprintf("localhost:%d", port)
+
+		switch cfg.clientTLS {
+		case clientNonTLS, clientTLS:
+			curl = (&url.URL{Scheme: cfg.clientScheme(), Host: curlHost}).String()
+			curls = []string{curl}
+		case clientTLSAndNonTLS:
+			curl = (&url.URL{Scheme: "http", Host: curlHost}).String()
+			curltls = (&url.URL{Scheme: "https", Host: curlHost}).String()
+			curls = []string{curl, curltls}
+		}
+
+		purl := url.URL{Scheme: cfg.peerScheme(), Host: fmt.Sprintf("localhost:%d", port+1)}
+		name := fmt.Sprintf("testname%d", i)
+		dataDirPath := cfg.dataDirPath
+		if cfg.dataDirPath == "" {
+			var derr error
+			dataDirPath, derr = ioutil.TempDir("", name+".etcd")
+			if derr != nil {
+				panic(fmt.Sprintf("could not get tempdir for datadir: %s", derr))
+			}
+		}
+		initialCluster[i] = fmt.Sprintf("%s=%s", name, purl.String())
+
+		args := []string{
+			"--name", name,
+			"--listen-client-urls", strings.Join(curls, ","),
+			"--advertise-client-urls", strings.Join(curls, ","),
+			"--listen-peer-urls", purl.String(),
+			"--initial-advertise-peer-urls", purl.String(),
+			"--initial-cluster-token", cfg.initialToken,
+			"--data-dir", dataDirPath,
+			"--snapshot-count", fmt.Sprintf("%d", cfg.snapshotCount),
+		}
+		args = addV2Args(args)
+		if cfg.forceNewCluster {
+			args = append(args, "--force-new-cluster")
+		}
+		if cfg.quotaBackendBytes > 0 {
+			args = append(args,
+				"--quota-backend-bytes", fmt.Sprintf("%d", cfg.quotaBackendBytes),
+			)
+		}
+		if cfg.noStrictReconfig {
+			args = append(args, "--strict-reconfig-check=false")
+		}
+		if cfg.enableV2 {
+			args = append(args, "--enable-v2")
+		}
+		if cfg.initialCorruptCheck {
+			args = append(args, "--experimental-initial-corrupt-check")
+		}
+		var murl string
+		if cfg.metricsURLScheme != "" {
+			murl = (&url.URL{
+				Scheme: cfg.metricsURLScheme,
+				Host:   fmt.Sprintf("localhost:%d", port+2),
+			}).String()
+			args = append(args, "--listen-metrics-urls", murl)
+		}
+
+		args = append(args, cfg.tlsArgs()...)
+
+		if cfg.authTokenOpts != "" {
+			args = append(args, "--auth-token", cfg.authTokenOpts)
+		}
+
+		etcdCfgs[i] = &etcdServerProcessConfig{
+			execPath:     cfg.execPath,
+			args:         args,
+			tlsArgs:      cfg.tlsArgs(),
+			dataDirPath:  dataDirPath,
+			keepDataDir:  cfg.keepDataDir,
+			name:         name,
+			purl:         purl,
+			acurl:        curl,
+			murl:         murl,
+			initialToken: cfg.initialToken,
+		}
+	}
+
+	initialClusterArgs := []string{"--initial-cluster", strings.Join(initialCluster, ",")}
+	for i := range etcdCfgs {
+		etcdCfgs[i].initialCluster = strings.Join(initialCluster, ",")
+		etcdCfgs[i].args = append(etcdCfgs[i].args, initialClusterArgs...)
+	}
+
+	return etcdCfgs
+}
+
+func (cfg *etcdProcessClusterConfig) tlsArgs() (args []string) {
+	if cfg.clientTLS != clientNonTLS {
+		if cfg.isClientAutoTLS {
+			args = append(args, "--auto-tls")
+		} else {
+			tlsClientArgs := []string{
+				"--cert-file", certPath,
+				"--key-file", privateKeyPath,
+				"--trusted-ca-file", caPath,
+			}
+			args = append(args, tlsClientArgs...)
+
+			if cfg.clientCertAuthEnabled {
+				args = append(args, "--client-cert-auth")
+			}
+		}
+	}
+
+	if cfg.isPeerTLS {
+		if cfg.isPeerAutoTLS {
+			args = append(args, "--peer-auto-tls")
+		} else {
+			tlsPeerArgs := []string{
+				"--peer-cert-file", certPath,
+				"--peer-key-file", privateKeyPath,
+				"--peer-trusted-ca-file", caPath,
+			}
+			args = append(args, tlsPeerArgs...)
+		}
+	}
+
+	if cfg.isClientCRL {
+		args = append(args, "--client-crl-file", crlPath, "--client-cert-auth")
+	}
+
+	if len(cfg.cipherSuites) > 0 {
+		args = append(args, "--cipher-suites", strings.Join(cfg.cipherSuites, ","))
+	}
+
+	return args
+}
+
+func (epc *etcdProcessCluster) EndpointsV2() []string {
+	return epc.endpoints(func(ep etcdProcess) []string { return ep.EndpointsV2() })
+}
+
+func (epc *etcdProcessCluster) EndpointsV3() []string {
+	return epc.endpoints(func(ep etcdProcess) []string { return ep.EndpointsV3() })
+}
+
+func (epc *etcdProcessCluster) endpoints(f func(ep etcdProcess) []string) (ret []string) {
+	for _, p := range epc.procs {
+		ret = append(ret, f(p)...)
+	}
+	return ret
+}
+
+func (epc *etcdProcessCluster) Start() error {
+	return epc.start(func(ep etcdProcess) error { return ep.Start() })
+}
+
+func (epc *etcdProcessCluster) Restart() error {
+	return epc.start(func(ep etcdProcess) error { return ep.Restart() })
+}
+
+func (epc *etcdProcessCluster) start(f func(ep etcdProcess) error) error {
+	readyC := make(chan error, len(epc.procs))
+	for i := range epc.procs {
+		go func(n int) { readyC <- f(epc.procs[n]) }(i)
+	}
+	for range epc.procs {
+		if err := <-readyC; err != nil {
+			epc.Close()
+			return err
+		}
+	}
+	return nil
+}
+
+func (epc *etcdProcessCluster) Stop() (err error) {
+	for _, p := range epc.procs {
+		if p == nil {
+			continue
+		}
+		if curErr := p.Stop(); curErr != nil {
+			if err != nil {
+				err = fmt.Errorf("%v; %v", err, curErr)
+			} else {
+				err = curErr
+			}
+		}
+	}
+	return err
+}
+
+func (epc *etcdProcessCluster) Close() error {
+	err := epc.Stop()
+	for _, p := range epc.procs {
+		// p is nil when newEtcdProcess fails in the middle
+		// Close still gets called to clean up test data
+		if p == nil {
+			continue
+		}
+		if cerr := p.Close(); cerr != nil {
+			err = cerr
+		}
+	}
+	return err
+}
+
+func (epc *etcdProcessCluster) WithStopSignal(sig os.Signal) (ret os.Signal) {
+	for _, p := range epc.procs {
+		ret = p.WithStopSignal(sig)
+	}
+	return ret
+}

tests/e2e/ctl_v2_test.go

@@ -0,0 +1,183 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/coreos/etcd/pkg/fileutil"
+	"github.com/coreos/etcd/pkg/testutil"
+)
+
+func testCtlV2Set(t *testing.T, cfg *etcdProcessClusterConfig, quorum bool) {
+	os.Setenv("ETCDCTL_API", "2")
+	defer os.Unsetenv("ETCDCTL_API")
+	defer testutil.AfterTest(t)
+
+	cfg.enableV2 = true
+	epc := setupEtcdctlTest(t, cfg, quorum)
+	defer func() {
+		if errC := epc.Close(); errC != nil {
+			t.Fatalf("error closing etcd processes (%v)", errC)
+		}
+	}()
+
+	key, value := "foo", "bar"
+
+	if err := etcdctlSet(epc, key, value); err != nil {
+		t.Fatalf("failed set (%v)", err)
+	}
+
+	if err := etcdctlGet(epc, key, value, quorum); err != nil {
+		t.Fatalf("failed get (%v)", err)
+	}
+}
+
+func etcdctlPrefixArgs(clus *etcdProcessCluster) []string {
+	endpoints := strings.Join(clus.EndpointsV2(), ",")
+	cmdArgs := []string{ctlBinPath, "--endpoints", endpoints}
+	if clus.cfg.clientTLS == clientTLS {
+		cmdArgs = append(cmdArgs, "--ca-file", caPath, "--cert-file", certPath, "--key-file", privateKeyPath)
+	}
+	return cmdArgs
+}
+
+func etcdctlClusterHealth(clus *etcdProcessCluster, val string) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "cluster-health")
+	return spawnWithExpect(cmdArgs, val)
+}
+
+func etcdctlSet(clus *etcdProcessCluster, key, value string) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "set", key, value)
+	return spawnWithExpect(cmdArgs, value)
+}
+
+func etcdctlMk(clus *etcdProcessCluster, key, value string, first bool) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "mk", key, value)
+	if first {
+		return spawnWithExpect(cmdArgs, value)
+	}
+	return spawnWithExpect(cmdArgs, "Error:  105: Key already exists")
+}
+
+func etcdctlGet(clus *etcdProcessCluster, key, value string, quorum bool) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "get", key)
+	if quorum {
+		cmdArgs = append(cmdArgs, "--quorum")
+	}
+	return spawnWithExpect(cmdArgs, value)
+}
+
+func etcdctlRm(clus *etcdProcessCluster, key, value string, first bool) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "rm", key)
+	if first {
+		return spawnWithExpect(cmdArgs, "PrevNode.Value: "+value)
+	}
+	return spawnWithExpect(cmdArgs, "Error:  100: Key not found")
+}
+
+func etcdctlLs(clus *etcdProcessCluster, key string, quorum bool) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "ls")
+	if quorum {
+		cmdArgs = append(cmdArgs, "--quorum")
+	}
+	return spawnWithExpect(cmdArgs, key)
+}
+
+func etcdctlWatch(clus *etcdProcessCluster, key, value string, noSync bool) <-chan error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "watch", "--after-index=1", key)
+	if noSync {
+		cmdArgs = append(cmdArgs, "--no-sync")
+	}
+	errc := make(chan error, 1)
+	go func() {
+		errc <- spawnWithExpect(cmdArgs, value)
+	}()
+	return errc
+}
+
+func etcdctlRoleAdd(clus *etcdProcessCluster, role string) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "role", "add", role)
+	return spawnWithExpect(cmdArgs, role)
+}
+
+func etcdctlRoleGrant(clus *etcdProcessCluster, role string, perms ...string) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "role", "grant")
+	cmdArgs = append(cmdArgs, perms...)
+	cmdArgs = append(cmdArgs, role)
+	return spawnWithExpect(cmdArgs, role)
+}
+
+func etcdctlRoleList(clus *etcdProcessCluster, expectedRole string) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "role", "list")
+	return spawnWithExpect(cmdArgs, expectedRole)
+}
+
+func etcdctlUserAdd(clus *etcdProcessCluster, user, pass string) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "user", "add", user+":"+pass)
+	return spawnWithExpect(cmdArgs, "User "+user+" created")
+}
+
+func etcdctlUserGrant(clus *etcdProcessCluster, user, role string) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "user", "grant", "--roles", role, user)
+	return spawnWithExpect(cmdArgs, "User "+user+" updated")
+}
+
+func etcdctlUserGet(clus *etcdProcessCluster, user string) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "user", "get", user)
+	return spawnWithExpect(cmdArgs, "User: "+user)
+}
+
+func etcdctlUserList(clus *etcdProcessCluster, expectedUser string) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "user", "list")
+	return spawnWithExpect(cmdArgs, expectedUser)
+}
+
+func etcdctlAuthEnable(clus *etcdProcessCluster) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "auth", "enable")
+	return spawnWithExpect(cmdArgs, "Authentication Enabled")
+}
+
+func etcdctlBackup(clus *etcdProcessCluster, dataDir, backupDir string, v3 bool) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "backup", "--data-dir", dataDir, "--backup-dir", backupDir)
+	if v3 {
+		cmdArgs = append(cmdArgs, "--with-v3")
+	}
+	proc, err := spawnCmd(cmdArgs)
+	if err != nil {
+		return err
+	}
+	return proc.Close()
+}
+
+func mustEtcdctl(t *testing.T) {
+	if !fileutil.Exist(binDir + "/etcdctl") {
+		t.Fatalf("could not find etcdctl binary")
+	}
+}
+
+func setupEtcdctlTest(t *testing.T, cfg *etcdProcessClusterConfig, quorum bool) *etcdProcessCluster {
+	mustEtcdctl(t)
+	if !quorum {
+		cfg = configStandalone(*cfg)
+	}
+	epc, err := newEtcdProcessCluster(cfg)
+	if err != nil {
+		t.Fatalf("could not start etcd process cluster (%v)", err)
+	}
+	return epc
+}

tests/e2e/ctl_v3_member_test.go

@@ -0,0 +1,154 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+	"testing"
+
+	"github.com/coreos/etcd/etcdserver/etcdserverpb"
+)
+
+func TestCtlV3MemberList(t *testing.T)          { testCtl(t, memberListTest) }
+func TestCtlV3MemberListNoTLS(t *testing.T)     { testCtl(t, memberListTest, withCfg(configNoTLS)) }
+func TestCtlV3MemberListClientTLS(t *testing.T) { testCtl(t, memberListTest, withCfg(configClientTLS)) }
+func TestCtlV3MemberListClientAutoTLS(t *testing.T) {
+	testCtl(t, memberListTest, withCfg(configClientAutoTLS))
+}
+func TestCtlV3MemberListPeerTLS(t *testing.T) { testCtl(t, memberListTest, withCfg(configPeerTLS)) }
+func TestCtlV3MemberRemove(t *testing.T) {
+	testCtl(t, memberRemoveTest, withQuorum(), withNoStrictReconfig())
+}
+func TestCtlV3MemberRemoveNoTLS(t *testing.T) {
+	testCtl(t, memberRemoveTest, withQuorum(), withNoStrictReconfig(), withCfg(configNoTLS))
+}
+func TestCtlV3MemberRemoveClientTLS(t *testing.T) {
+	testCtl(t, memberRemoveTest, withQuorum(), withNoStrictReconfig(), withCfg(configClientTLS))
+}
+func TestCtlV3MemberRemoveClientAutoTLS(t *testing.T) {
+	testCtl(t, memberRemoveTest, withQuorum(), withNoStrictReconfig(), withCfg(
+		// default clusterSize is 1
+		etcdProcessClusterConfig{
+			clusterSize:     3,
+			isClientAutoTLS: true,
+			clientTLS:       clientTLS,
+			initialToken:    "new",
+		}))
+}
+func TestCtlV3MemberRemovePeerTLS(t *testing.T) {
+	testCtl(t, memberRemoveTest, withQuorum(), withNoStrictReconfig(), withCfg(configPeerTLS))
+}
+func TestCtlV3MemberAdd(t *testing.T)          { testCtl(t, memberAddTest) }
+func TestCtlV3MemberAddNoTLS(t *testing.T)     { testCtl(t, memberAddTest, withCfg(configNoTLS)) }
+func TestCtlV3MemberAddClientTLS(t *testing.T) { testCtl(t, memberAddTest, withCfg(configClientTLS)) }
+func TestCtlV3MemberAddClientAutoTLS(t *testing.T) {
+	testCtl(t, memberAddTest, withCfg(configClientAutoTLS))
+}
+func TestCtlV3MemberAddPeerTLS(t *testing.T)  { testCtl(t, memberAddTest, withCfg(configPeerTLS)) }
+func TestCtlV3MemberUpdate(t *testing.T)      { testCtl(t, memberUpdateTest) }
+func TestCtlV3MemberUpdateNoTLS(t *testing.T) { testCtl(t, memberUpdateTest, withCfg(configNoTLS)) }
+func TestCtlV3MemberUpdateClientTLS(t *testing.T) {
+	testCtl(t, memberUpdateTest, withCfg(configClientTLS))
+}
+func TestCtlV3MemberUpdateClientAutoTLS(t *testing.T) {
+	testCtl(t, memberUpdateTest, withCfg(configClientAutoTLS))
+}
+func TestCtlV3MemberUpdatePeerTLS(t *testing.T) { testCtl(t, memberUpdateTest, withCfg(configPeerTLS)) }
+
+func memberListTest(cx ctlCtx) {
+	if err := ctlV3MemberList(cx); err != nil {
+		cx.t.Fatalf("memberListTest ctlV3MemberList error (%v)", err)
+	}
+}
+
+func ctlV3MemberList(cx ctlCtx) error {
+	cmdArgs := append(cx.PrefixArgs(), "member", "list")
+	lines := make([]string, cx.cfg.clusterSize)
+	for i := range lines {
+		lines[i] = "started"
+	}
+	return spawnWithExpects(cmdArgs, lines...)
+}
+
+func getMemberList(cx ctlCtx) (etcdserverpb.MemberListResponse, error) {
+	cmdArgs := append(cx.PrefixArgs(), "--write-out", "json", "member", "list")
+
+	proc, err := spawnCmd(cmdArgs)
+	if err != nil {
+		return etcdserverpb.MemberListResponse{}, err
+	}
+	var txt string
+	txt, err = proc.Expect("members")
+	if err != nil {
+		return etcdserverpb.MemberListResponse{}, err
+	}
+	if err = proc.Close(); err != nil {
+		return etcdserverpb.MemberListResponse{}, err
+	}
+
+	resp := etcdserverpb.MemberListResponse{}
+	dec := json.NewDecoder(strings.NewReader(txt))
+	if err := dec.Decode(&resp); err == io.EOF {
+		return etcdserverpb.MemberListResponse{}, err
+	}
+	return resp, nil
+}
+
+func memberRemoveTest(cx ctlCtx) {
+	ep, memIDToRemove, clusterID := cx.memberToRemove()
+	if err := ctlV3MemberRemove(cx, ep, memIDToRemove, clusterID); err != nil {
+		cx.t.Fatal(err)
+	}
+}
+
+func ctlV3MemberRemove(cx ctlCtx, ep, memberID, clusterID string) error {
+	cmdArgs := append(cx.prefixArgs([]string{ep}), "member", "remove", memberID)
+	return spawnWithExpect(cmdArgs, fmt.Sprintf("%s removed from cluster %s", memberID, clusterID))
+}
+
+func memberAddTest(cx ctlCtx) {
+	if err := ctlV3MemberAdd(cx, fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+11), false); err != nil {
+		cx.t.Fatal(err)
+	}
+}
+
+func ctlV3MemberAdd(cx ctlCtx, peerURL string, isLearner bool) error {
+	cmdArgs := append(cx.PrefixArgs(), "member", "add", "newmember", fmt.Sprintf("--peer-urls=%s", peerURL))
+	if isLearner {
+		cmdArgs = append(cmdArgs, "--learner")
+	}
+	return spawnWithExpect(cmdArgs, " added to cluster ")
+}
+
+func memberUpdateTest(cx ctlCtx) {
+	mr, err := getMemberList(cx)
+	if err != nil {
+		cx.t.Fatal(err)
+	}
+
+	peerURL := fmt.Sprintf("http://localhost:%d", etcdProcessBasePort+11)
+	memberID := fmt.Sprintf("%x", mr.Members[0].ID)
+	if err = ctlV3MemberUpdate(cx, memberID, peerURL); err != nil {
+		cx.t.Fatal(err)
+	}
+}
+
+func ctlV3MemberUpdate(cx ctlCtx, memberID, peerURL string) error {
+	cmdArgs := append(cx.PrefixArgs(), "member", "update", memberID, fmt.Sprintf("--peer-urls=%s", peerURL))
+	return spawnWithExpect(cmdArgs, " updated in cluster ")
+}

tests/e2e/ctl_v3_test.go

@@ -0,0 +1,255 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coreos/etcd/pkg/flags"
+	"github.com/coreos/etcd/pkg/testutil"
+	"github.com/coreos/etcd/version"
+)
+
+func TestCtlV3Version(t *testing.T) { testCtl(t, versionTest) }
+
+func versionTest(cx ctlCtx) {
+	if err := ctlV3Version(cx); err != nil {
+		cx.t.Fatalf("versionTest ctlV3Version error (%v)", err)
+	}
+}
+
+func ctlV3Version(cx ctlCtx) error {
+	cmdArgs := append(cx.PrefixArgs(), "version")
+	return spawnWithExpect(cmdArgs, version.Version)
+}
+
+// TestCtlV3DialWithHTTPScheme ensures that client handles endpoints with HTTPS scheme.
+func TestCtlV3DialWithHTTPScheme(t *testing.T) {
+	testCtl(t, dialWithSchemeTest, withCfg(configClientTLS))
+}
+
+func dialWithSchemeTest(cx ctlCtx) {
+	cmdArgs := append(cx.prefixArgs(cx.epc.EndpointsV3()), "put", "foo", "bar")
+	if err := spawnWithExpect(cmdArgs, "OK"); err != nil {
+		cx.t.Fatal(err)
+	}
+}
+
+type ctlCtx struct {
+	t                 *testing.T
+	apiPrefix         string
+	cfg               etcdProcessClusterConfig
+	quotaBackendBytes int64
+	corruptFunc       func(string) error
+	noStrictReconfig  bool
+
+	epc *etcdProcessCluster
+
+	envMap map[string]struct{}
+
+	dialTimeout time.Duration
+
+	quorum      bool // if true, set up 3-node cluster and linearizable read
+	interactive bool
+
+	user string
+	pass string
+
+	initialCorruptCheck bool
+
+	// for compaction
+	compactPhysical bool
+}
+
+type ctlOption func(*ctlCtx)
+
+func (cx *ctlCtx) applyOpts(opts []ctlOption) {
+	for _, opt := range opts {
+		opt(cx)
+	}
+	cx.initialCorruptCheck = true
+}
+
+func withCfg(cfg etcdProcessClusterConfig) ctlOption {
+	return func(cx *ctlCtx) { cx.cfg = cfg }
+}
+
+func withDialTimeout(timeout time.Duration) ctlOption {
+	return func(cx *ctlCtx) { cx.dialTimeout = timeout }
+}
+
+func withQuorum() ctlOption {
+	return func(cx *ctlCtx) { cx.quorum = true }
+}
+
+func withInteractive() ctlOption {
+	return func(cx *ctlCtx) { cx.interactive = true }
+}
+
+func withQuota(b int64) ctlOption {
+	return func(cx *ctlCtx) { cx.quotaBackendBytes = b }
+}
+
+func withCompactPhysical() ctlOption {
+	return func(cx *ctlCtx) { cx.compactPhysical = true }
+}
+
+func withInitialCorruptCheck() ctlOption {
+	return func(cx *ctlCtx) { cx.initialCorruptCheck = true }
+}
+
+func withCorruptFunc(f func(string) error) ctlOption {
+	return func(cx *ctlCtx) { cx.corruptFunc = f }
+}
+
+func withNoStrictReconfig() ctlOption {
+	return func(cx *ctlCtx) { cx.noStrictReconfig = true }
+}
+
+func withApiPrefix(p string) ctlOption {
+	return func(cx *ctlCtx) { cx.apiPrefix = p }
+}
+
+func withFlagByEnv() ctlOption {
+	return func(cx *ctlCtx) { cx.envMap = make(map[string]struct{}) }
+}
+
+func testCtl(t *testing.T, testFunc func(ctlCtx), opts ...ctlOption) {
+	defer testutil.AfterTest(t)
+
+	ret := ctlCtx{
+		t:           t,
+		cfg:         configAutoTLS,
+		dialTimeout: 7 * time.Second,
+	}
+	ret.applyOpts(opts)
+
+	mustEtcdctl(t)
+	if !ret.quorum {
+		ret.cfg = *configStandalone(ret.cfg)
+	}
+	if ret.quotaBackendBytes > 0 {
+		ret.cfg.quotaBackendBytes = ret.quotaBackendBytes
+	}
+	ret.cfg.noStrictReconfig = ret.noStrictReconfig
+	if ret.initialCorruptCheck {
+		ret.cfg.initialCorruptCheck = ret.initialCorruptCheck
+	}
+
+	epc, err := newEtcdProcessCluster(&ret.cfg)
+	if err != nil {
+		t.Fatalf("could not start etcd process cluster (%v)", err)
+	}
+	ret.epc = epc
+
+	defer func() {
+		if ret.envMap != nil {
+			for k := range ret.envMap {
+				os.Unsetenv(k)
+			}
+		}
+		if errC := ret.epc.Close(); errC != nil {
+			t.Fatalf("error closing etcd processes (%v)", errC)
+		}
+	}()
+
+	donec := make(chan struct{})
+	go func() {
+		defer close(donec)
+		testFunc(ret)
+	}()
+
+	timeout := 2*ret.dialTimeout + time.Second
+	if ret.dialTimeout == 0 {
+		timeout = 30 * time.Second
+	}
+	select {
+	case <-time.After(timeout):
+		testutil.FatalStack(t, fmt.Sprintf("test timed out after %v", timeout))
+	case <-donec:
+	}
+}
+
+func (cx *ctlCtx) prefixArgs(eps []string) []string {
+	fmap := make(map[string]string)
+	fmap["endpoints"] = strings.Join(eps, ",")
+	fmap["dial-timeout"] = cx.dialTimeout.String()
+	if cx.epc.cfg.clientTLS == clientTLS {
+		if cx.epc.cfg.isClientAutoTLS {
+			fmap["insecure-transport"] = "false"
+			fmap["insecure-skip-tls-verify"] = "true"
+		} else if cx.epc.cfg.isClientCRL {
+			fmap["cacert"] = caPath
+			fmap["cert"] = revokedCertPath
+			fmap["key"] = revokedPrivateKeyPath
+		} else {
+			fmap["cacert"] = caPath
+			fmap["cert"] = certPath
+			fmap["key"] = privateKeyPath
+		}
+	}
+	if cx.user != "" {
+		fmap["user"] = cx.user + ":" + cx.pass
+	}
+
+	useEnv := cx.envMap != nil
+
+	cmdArgs := []string{ctlBinPath + "3"}
+	for k, v := range fmap {
+		if useEnv {
+			ek := flags.FlagToEnv("ETCDCTL", k)
+			os.Setenv(ek, v)
+			cx.envMap[ek] = struct{}{}
+		} else {
+			cmdArgs = append(cmdArgs, fmt.Sprintf("--%s=%s", k, v))
+		}
+	}
+	return cmdArgs
+}
+
+// PrefixArgs prefixes etcdctl command.
+// Make sure to unset environment variables after tests.
+func (cx *ctlCtx) PrefixArgs() []string {
+	return cx.prefixArgs(cx.epc.EndpointsV3())
+}
+
+func isGRPCTimedout(err error) bool {
+	return strings.Contains(err.Error(), "grpc: timed out trying to connect")
+}
+
+func (cx *ctlCtx) memberToRemove() (ep string, memberID string, clusterID string) {
+	n1 := cx.cfg.clusterSize
+	if n1 < 2 {
+		cx.t.Fatalf("%d-node is too small to test 'member remove'", n1)
+	}
+
+	resp, err := getMemberList(*cx)
+	if err != nil {
+		cx.t.Fatal(err)
+	}
+	if n1 != len(resp.Members) {
+		cx.t.Fatalf("expected %d, got %d", n1, len(resp.Members))
+	}
+
+	ep = resp.Members[0].ClientURLs[0]
+	clusterID = fmt.Sprintf("%x", resp.Header.ClusterId)
+	memberID = fmt.Sprintf("%x", resp.Members[1].ID)
+
+	return ep, memberID, clusterID
+}

tests/e2e/ctl_v3_watch_cov_test.go

@@ -1,134 +0,0 @@
-// Copyright 2018 The etcd Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// +build cov
-
-package e2e
-
-import (
-	"os"
-	"testing"
-)
-
-func TestCtlV3Watch(t *testing.T)          { testCtl(t, watchTest) }
-func TestCtlV3WatchNoTLS(t *testing.T)     { testCtl(t, watchTest, withCfg(configNoTLS)) }
-func TestCtlV3WatchClientTLS(t *testing.T) { testCtl(t, watchTest, withCfg(configClientTLS)) }
-func TestCtlV3WatchPeerTLS(t *testing.T)   { testCtl(t, watchTest, withCfg(configPeerTLS)) }
-func TestCtlV3WatchTimeout(t *testing.T)   { testCtl(t, watchTest, withDialTimeout(0)) }
-
-func TestCtlV3WatchInteractive(t *testing.T) {
-	testCtl(t, watchTest, withInteractive())
-}
-func TestCtlV3WatchInteractiveNoTLS(t *testing.T) {
-	testCtl(t, watchTest, withInteractive(), withCfg(configNoTLS))
-}
-func TestCtlV3WatchInteractiveClientTLS(t *testing.T) {
-	testCtl(t, watchTest, withInteractive(), withCfg(configClientTLS))
-}
-func TestCtlV3WatchInteractivePeerTLS(t *testing.T) {
-	testCtl(t, watchTest, withInteractive(), withCfg(configPeerTLS))
-}
-
-func watchTest(cx ctlCtx) {
-	tests := []struct {
-		puts     []kv
-		envKey   string
-		envRange string
-		args     []string
-
-		wkv []kvExec
-	}{
-		{ // watch 1 key
-			puts: []kv{{"sample", "value"}},
-			args: []string{"sample", "--rev", "1"},
-			wkv:  []kvExec{{key: "sample", val: "value"}},
-		},
-		{ // watch 1 key with env
-			puts:   []kv{{"sample", "value"}},
-			envKey: "sample",
-			args:   []string{"--rev", "1"},
-			wkv:    []kvExec{{key: "sample", val: "value"}},
-		},
-
-		// coverage tests get extra arguments:
-		// ./bin/etcdctl_test -test.coverprofile=e2e.1525392462795198897.coverprofile -test.outputdir=../..
-		// do not test watch exec commands
-
-		{ // watch 3 keys by prefix
-			puts: []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
-			args: []string{"key", "--rev", "1", "--prefix"},
-			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
-		},
-		{ // watch 3 keys by prefix, with env
-			puts:   []kv{{"key1", "val1"}, {"key2", "val2"}, {"key3", "val3"}},
-			envKey: "key",
-			args:   []string{"--rev", "1", "--prefix"},
-			wkv:    []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}, {key: "key3", val: "val3"}},
-		},
-		{ // watch by revision
-			puts: []kv{{"etcd", "revision_1"}, {"etcd", "revision_2"}, {"etcd", "revision_3"}},
-			args: []string{"etcd", "--rev", "2"},
-			wkv:  []kvExec{{key: "etcd", val: "revision_2"}, {key: "etcd", val: "revision_3"}},
-		},
-		{ // watch 3 keys by range
-			puts: []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
-			args: []string{"key", "key3", "--rev", "1"},
-			wkv:  []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
-		},
-		{ // watch 3 keys by range, with env
-			puts:     []kv{{"key1", "val1"}, {"key3", "val3"}, {"key2", "val2"}},
-			envKey:   "key",
-			envRange: "key3",
-			args:     []string{"--rev", "1"},
-			wkv:      []kvExec{{key: "key1", val: "val1"}, {key: "key2", val: "val2"}},
-		},
-	}
-
-	for i, tt := range tests {
-		donec := make(chan struct{})
-		go func(i int, puts []kv) {
-			for j := range puts {
-				if err := ctlV3Put(cx, puts[j].key, puts[j].val, ""); err != nil {
-					cx.t.Fatalf("watchTest #%d-%d: ctlV3Put error (%v)", i, j, err)
-				}
-			}
-			close(donec)
-		}(i, tt.puts)
-
-		unsetEnv := func() {}
-		if tt.envKey != "" || tt.envRange != "" {
-			if tt.envKey != "" {
-				os.Setenv("ETCDCTL_WATCH_KEY", tt.envKey)
-				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_KEY") }
-			}
-			if tt.envRange != "" {
-				os.Setenv("ETCDCTL_WATCH_RANGE_END", tt.envRange)
-				unsetEnv = func() { os.Unsetenv("ETCDCTL_WATCH_RANGE_END") }
-			}
-			if tt.envKey != "" && tt.envRange != "" {
-				unsetEnv = func() {
-					os.Unsetenv("ETCDCTL_WATCH_KEY")
-					os.Unsetenv("ETCDCTL_WATCH_RANGE_END")
-				}
-			}
-		}
-		if err := ctlV3Watch(cx, tt.args, tt.wkv...); err != nil {
-			if cx.dialTimeout > 0 && !isGRPCTimedout(err) {
-				cx.t.Errorf("watchTest #%d: ctlV3Watch error (%v)", i, err)
-			}
-		}
-		unsetEnv()
-		<-donec
-	}
-}

tests/e2e/ctl_v3_watch_test.go

@@ -0,0 +1,149 @@
+// Copyright 2018 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"strings"
+)
+
+type kvExec struct {
+	key, val   string
+	execOutput string
+}
+
+func setupWatchArgs(cx ctlCtx, args []string) []string {
+	cmdArgs := append(cx.PrefixArgs(), "watch")
+	if cx.interactive {
+		cmdArgs = append(cmdArgs, "--interactive")
+	} else {
+		cmdArgs = append(cmdArgs, args...)
+	}
+
+	return cmdArgs
+}
+
+func ctlV3Watch(cx ctlCtx, args []string, kvs ...kvExec) error {
+	cmdArgs := setupWatchArgs(cx, args)
+
+	proc, err := spawnCmd(cmdArgs)
+	if err != nil {
+		return err
+	}
+
+	if cx.interactive {
+		wl := strings.Join(append([]string{"watch"}, args...), " ") + "\r"
+		if err = proc.Send(wl); err != nil {
+			return err
+		}
+	}
+
+	for _, elem := range kvs {
+		if _, err = proc.Expect(elem.key); err != nil {
+			return err
+		}
+		if _, err = proc.Expect(elem.val); err != nil {
+			return err
+		}
+		if elem.execOutput != "" {
+			if _, err = proc.Expect(elem.execOutput); err != nil {
+				return err
+			}
+		}
+	}
+	return proc.Stop()
+}
+
+func ctlV3WatchFailPerm(cx ctlCtx, args []string) error {
+	cmdArgs := setupWatchArgs(cx, args)
+
+	proc, err := spawnCmd(cmdArgs)
+	if err != nil {
+		return err
+	}
+
+	if cx.interactive {
+		wl := strings.Join(append([]string{"watch"}, args...), " ") + "\r"
+		if err = proc.Send(wl); err != nil {
+			return err
+		}
+	}
+
+	// TODO(mitake): after printing accurate error message that includes
+	// "permission denied", the above string argument of proc.Expect()
+	// should be updated.
+	_, err = proc.Expect("watch is canceled by the server")
+	if err != nil {
+		return err
+	}
+	return proc.Close()
+}
+
+func ctlV3Put(cx ctlCtx, key, value, leaseID string, flags ...string) error {
+	skipValue := false
+	skipLease := false
+	for _, f := range flags {
+		if f == "--ignore-value" {
+			skipValue = true
+		}
+		if f == "--ignore-lease" {
+			skipLease = true
+		}
+	}
+	cmdArgs := append(cx.PrefixArgs(), "put", key)
+	if !skipValue {
+		cmdArgs = append(cmdArgs, value)
+	}
+	if leaseID != "" && !skipLease {
+		cmdArgs = append(cmdArgs, "--lease", leaseID)
+	}
+	if len(flags) != 0 {
+		cmdArgs = append(cmdArgs, flags...)
+	}
+	return spawnWithExpect(cmdArgs, "OK")
+}
+
+type kv struct {
+	key, val string
+}
+
+func ctlV3Get(cx ctlCtx, args []string, kvs ...kv) error {
+	cmdArgs := append(cx.PrefixArgs(), "get")
+	cmdArgs = append(cmdArgs, args...)
+	if !cx.quorum {
+		cmdArgs = append(cmdArgs, "--consistency", "s")
+	}
+	var lines []string
+	for _, elem := range kvs {
+		lines = append(lines, elem.key, elem.val)
+	}
+	return spawnWithExpects(cmdArgs, lines...)
+}
+
+// ctlV3GetWithErr runs "get" command expecting no output but error
+func ctlV3GetWithErr(cx ctlCtx, args []string, errs []string) error {
+	cmdArgs := append(cx.PrefixArgs(), "get")
+	cmdArgs = append(cmdArgs, args...)
+	if !cx.quorum {
+		cmdArgs = append(cmdArgs, "--consistency", "s")
+	}
+	return spawnWithExpects(cmdArgs, errs...)
+}
+
+func ctlV3Del(cx ctlCtx, args []string, num int) error {
+	cmdArgs := append(cx.PrefixArgs(), "del")
+	cmdArgs = append(cmdArgs, args...)
+	return spawnWithExpects(cmdArgs, fmt.Sprintf("%d", num))
+}

tests/e2e/etcd_process.go

@@ -0,0 +1,145 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"fmt"
+	"net/url"
+	"os"
+
+	"github.com/coreos/etcd/pkg/expect"
+	"github.com/coreos/etcd/pkg/fileutil"
+)
+
+var (
+	etcdServerReadyLines = []string{"enabled capabilities for version", "published"}
+	binPath              string
+	ctlBinPath           string
+)
+
+// etcdProcess is a process that serves etcd requests.
+type etcdProcess interface {
+	EndpointsV2() []string
+	EndpointsV3() []string
+	EndpointsMetrics() []string
+
+	Start() error
+	Restart() error
+	Stop() error
+	Close() error
+	WithStopSignal(sig os.Signal) os.Signal
+	Config() *etcdServerProcessConfig
+}
+
+type etcdServerProcess struct {
+	cfg   *etcdServerProcessConfig
+	proc  *expect.ExpectProcess
+	donec chan struct{} // closed when Interact() terminates
+}
+
+type etcdServerProcessConfig struct {
+	execPath string
+	args     []string
+	tlsArgs  []string
+
+	dataDirPath string
+	keepDataDir bool
+
+	name string
+
+	purl url.URL
+
+	acurl string
+	murl  string
+
+	initialToken   string
+	initialCluster string
+}
+
+func newEtcdServerProcess(cfg *etcdServerProcessConfig) (*etcdServerProcess, error) {
+	if !fileutil.Exist(cfg.execPath) {
+		return nil, fmt.Errorf("could not find etcd binary")
+	}
+	if !cfg.keepDataDir {
+		if err := os.RemoveAll(cfg.dataDirPath); err != nil {
+			return nil, err
+		}
+	}
+	return &etcdServerProcess{cfg: cfg, donec: make(chan struct{})}, nil
+}
+
+func (ep *etcdServerProcess) EndpointsV2() []string      { return []string{ep.cfg.acurl} }
+func (ep *etcdServerProcess) EndpointsV3() []string      { return ep.EndpointsV2() }
+func (ep *etcdServerProcess) EndpointsMetrics() []string { return []string{ep.cfg.murl} }
+
+func (ep *etcdServerProcess) Start() error {
+	if ep.proc != nil {
+		panic("already started")
+	}
+	proc, err := spawnCmd(append([]string{ep.cfg.execPath}, ep.cfg.args...))
+	if err != nil {
+		return err
+	}
+	ep.proc = proc
+	return ep.waitReady()
+}
+
+func (ep *etcdServerProcess) Restart() error {
+	if err := ep.Stop(); err != nil {
+		return err
+	}
+	ep.donec = make(chan struct{})
+	return ep.Start()
+}
+
+func (ep *etcdServerProcess) Stop() (err error) {
+	if ep == nil || ep.proc == nil {
+		return nil
+	}
+	err = ep.proc.Stop()
+	if err != nil {
+		return err
+	}
+	ep.proc = nil
+	<-ep.donec
+	ep.donec = make(chan struct{})
+	if ep.cfg.purl.Scheme == "unix" || ep.cfg.purl.Scheme == "unixs" {
+		err = os.Remove(ep.cfg.purl.Host + ep.cfg.purl.Path)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (ep *etcdServerProcess) Close() error {
+	if err := ep.Stop(); err != nil {
+		return err
+	}
+	return os.RemoveAll(ep.cfg.dataDirPath)
+}
+
+func (ep *etcdServerProcess) WithStopSignal(sig os.Signal) os.Signal {
+	ret := ep.proc.StopSignal
+	ep.proc.StopSignal = sig
+	return ret
+}
+
+func (ep *etcdServerProcess) waitReady() error {
+	defer close(ep.donec)
+	return waitReadyExpectProc(ep.proc, etcdServerReadyLines)
+}
+
+func (ep *etcdServerProcess) Config() *etcdServerProcessConfig { return ep.cfg }

tests/e2e/etcd_spawn_nocov.go

@@ -0,0 +1,33 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !cov
+
+package e2e
+
+import (
+	"os"
+
+	"github.com/coreos/etcd/pkg/expect"
+)
+
+const noOutputLineCount = 0 // regular binaries emit no extra lines
+
+func spawnCmd(args []string) (*expect.ExpectProcess, error) {
+	if args[0] == ctlBinPath+"3" {
+		env := append(os.Environ(), "ETCDCTL_API=3")
+		return expect.NewExpectWithEnv(ctlBinPath, args[1:], env)
+	}
+	return expect.NewExpect(args[0], args[1:]...)
+}

tests/e2e/main_test.go

@@ -0,0 +1,63 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package e2e
+
+import (
+	"flag"
+	"os"
+	"runtime"
+	"testing"
+
+	"github.com/coreos/etcd/pkg/testutil"
+)
+
+var (
+	binDir  string
+	certDir string
+
+	certPath       string
+	privateKeyPath string
+	caPath         string
+
+	certPath2       string
+	privateKeyPath2 string
+
+	certPath3       string
+	privateKeyPath3 string
+
+	crlPath               string
+	revokedCertPath       string
+	revokedPrivateKeyPath string
+)
+
+func TestMain(m *testing.M) {
+	os.Setenv("ETCD_UNSUPPORTED_ARCH", runtime.GOARCH)
+	os.Unsetenv("ETCDCTL_API")
+
+	flag.StringVar(&binDir, "bin-dir", "../../bin", "The directory for store etcd and etcdctl binaries.")
+	flag.StringVar(&certDir, "cert-dir", "../../integration/fixtures", "The directory for store certificate files.")
+	flag.Parse()
+
+	binPath = binDir + "/etcd"
+	ctlBinPath = binDir + "/etcdctl"
+	certPath = certDir + "/server.crt"
+	privateKeyPath = certDir + "/server.key.insecure"
+	caPath = certDir + "/ca.crt"
+	revokedCertPath = certDir + "/server-revoked.crt"
+	revokedPrivateKeyPath = certDir + "/server-revoked.key.insecure"
+	crlPath = certDir + "/revoke.crl"
+
+	certPath2 = certDir + "/server2.crt"
+	privateKeyPath2 = certDir + "/server2.key.insecure"
+
+	certPath3 = certDir + "/server3.crt"
+	privateKeyPath3 = certDir + "/server3.key.insecure"
+
+	v := m.Run()
+	if v == 0 && testutil.CheckLeakedGoroutine() {
+		os.Exit(1)
+	}
+	os.Exit(v)
+}

tests/e2e/util.go

@@ -0,0 +1,110 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"strings"
+	"time"
+
+	"github.com/coreos/etcd/pkg/expect"
+)
+
+func waitReadyExpectProc(exproc *expect.ExpectProcess, readyStrs []string) error {
+	c := 0
+	matchSet := func(l string) bool {
+		for _, s := range readyStrs {
+			if strings.Contains(l, s) {
+				c++
+				break
+			}
+		}
+		return c == len(readyStrs)
+	}
+	_, err := exproc.ExpectFunc(matchSet)
+	return err
+}
+
+func spawnWithExpect(args []string, expected string) error {
+	return spawnWithExpects(args, []string{expected}...)
+}
+
+func spawnWithExpects(args []string, xs ...string) error {
+	_, err := spawnWithExpectLines(args, xs...)
+	return err
+}
+
+func spawnWithExpectLines(args []string, xs ...string) ([]string, error) {
+	proc, err := spawnCmd(args)
+	if err != nil {
+		return nil, err
+	}
+	// process until either stdout or stderr contains
+	// the expected string
+	var (
+		lines    []string
+		lineFunc = func(txt string) bool { return true }
+	)
+	for _, txt := range xs {
+		for {
+			l, lerr := proc.ExpectFunc(lineFunc)
+			if lerr != nil {
+				proc.Close()
+				return nil, fmt.Errorf("%v (expected %q, got %q)", lerr, txt, lines)
+			}
+			lines = append(lines, l)
+			if strings.Contains(l, txt) {
+				break
+			}
+		}
+	}
+	perr := proc.Close()
+	if len(xs) == 0 && proc.LineCount() != noOutputLineCount { // expect no output
+		return nil, fmt.Errorf("unexpected output (got lines %q, line count %d)", lines, proc.LineCount())
+	}
+	return lines, perr
+}
+
+func randomLeaseID() int64 {
+	return rand.New(rand.NewSource(time.Now().UnixNano())).Int63()
+}
+
+func dataMarshal(data interface{}) (d string, e error) {
+	m, err := json.Marshal(data)
+	if err != nil {
+		return "", err
+	}
+	return string(m), nil
+}
+
+func closeWithTimeout(p *expect.ExpectProcess, d time.Duration) error {
+	errc := make(chan error, 1)
+	go func() { errc <- p.Close() }()
+	select {
+	case err := <-errc:
+		return err
+	case <-time.After(d):
+		p.Stop()
+		// retry close after stopping to collect SIGQUIT data, if any
+		closeWithTimeout(p, time.Second)
+	}
+	return fmt.Errorf("took longer than %v to Close process %+v", d, p)
+}
+
+func toTLS(s string) string {
+	return strings.Replace(s, "http://", "https://", 1)
+}

tests/e2e/v2_test.go

@@ -0,0 +1,19 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !v2v3
+
+package e2e
+
+func addV2Args(args []string) []string { return args }