e2e: quorum cleanup

If a test gets data without quorum, it should only have one node in the
test cluster to avoid reading stale data.

Fixes #4694

e2e/etcd_test.go

@@ -35,49 +35,49 @@ const (
 )
 
 var (
-	defaultConfig = etcdProcessClusterConfig{
+	configNoTLS = etcdProcessClusterConfig{
 		clusterSize:  3,
 		proxySize:    0,
 		isClientTLS:  false,
 		isPeerTLS:    false,
 		initialToken: "new",
 	}
-	defaultConfigTLS = etcdProcessClusterConfig{
+	configTLS = etcdProcessClusterConfig{
 		clusterSize:  3,
 		proxySize:    0,
 		isClientTLS:  true,
 		isPeerTLS:    true,
 		initialToken: "new",
 	}
-	defaultConfigClientTLS = etcdProcessClusterConfig{
+	configClientTLS = etcdProcessClusterConfig{
 		clusterSize:  3,
 		proxySize:    0,
 		isClientTLS:  true,
 		isPeerTLS:    false,
 		initialToken: "new",
 	}
-	defaultConfigPeerTLS = etcdProcessClusterConfig{
+	configPeerTLS = etcdProcessClusterConfig{
 		clusterSize:  3,
 		proxySize:    0,
 		isClientTLS:  false,
 		isPeerTLS:    true,
 		initialToken: "new",
 	}
-	defaultConfigWithProxy = etcdProcessClusterConfig{
+	configWithProxy = etcdProcessClusterConfig{
 		clusterSize:  3,
 		proxySize:    1,
 		isClientTLS:  false,
 		isPeerTLS:    false,
 		initialToken: "new",
 	}
-	defaultConfigWithProxyTLS = etcdProcessClusterConfig{
+	configWithProxyTLS = etcdProcessClusterConfig{
 		clusterSize:  3,
 		proxySize:    1,
 		isClientTLS:  true,
 		isPeerTLS:    true,
 		initialToken: "new",
 	}
-	defaultConfigWithProxyPeerTLS = etcdProcessClusterConfig{
+	configWithProxyPeerTLS = etcdProcessClusterConfig{
 		clusterSize:  3,
 		proxySize:    1,
 		isClientTLS:  false,
@@ -86,17 +86,27 @@ var (
 	}
 )
 
-func TestBasicOpsNoTLS(t *testing.T)        { testBasicOpsPutGet(t, &defaultConfig) }
-func TestBasicOpsAllTLS(t *testing.T)       { testBasicOpsPutGet(t, &defaultConfigTLS) }
-func TestBasicOpsPeerTLS(t *testing.T)      { testBasicOpsPutGet(t, &defaultConfigPeerTLS) }
-func TestBasicOpsClientTLS(t *testing.T)    { testBasicOpsPutGet(t, &defaultConfigClientTLS) }
-func TestBasicOpsProxyNoTLS(t *testing.T)   { testBasicOpsPutGet(t, &defaultConfigWithProxy) }
-func TestBasicOpsProxyTLS(t *testing.T)     { testBasicOpsPutGet(t, &defaultConfigWithProxyTLS) }
-func TestBasicOpsProxyPeerTLS(t *testing.T) { testBasicOpsPutGet(t, &defaultConfigWithProxyPeerTLS) }
+func configStandalone(cfg etcdProcessClusterConfig) *etcdProcessClusterConfig {
+	ret := cfg
+	ret.clusterSize = 1
+	return &ret
+}
+
+func TestBasicOpsNoTLS(t *testing.T)        { testBasicOpsPutGet(t, &configNoTLS) }
+func TestBasicOpsAllTLS(t *testing.T)       { testBasicOpsPutGet(t, &configTLS) }
+func TestBasicOpsPeerTLS(t *testing.T)      { testBasicOpsPutGet(t, &configPeerTLS) }
+func TestBasicOpsClientTLS(t *testing.T)    { testBasicOpsPutGet(t, &configClientTLS) }
+func TestBasicOpsProxyNoTLS(t *testing.T)   { testBasicOpsPutGet(t, &configWithProxy) }
+func TestBasicOpsProxyTLS(t *testing.T)     { testBasicOpsPutGet(t, &configWithProxyTLS) }
+func TestBasicOpsProxyPeerTLS(t *testing.T) { testBasicOpsPutGet(t, &configWithProxyPeerTLS) }
 
 func testBasicOpsPutGet(t *testing.T, cfg *etcdProcessClusterConfig) {
 	defer testutil.AfterTest(t)
 
+	// test doesn't use quorum gets, so ensure there are no followers to avoid
+	// stale reads that will break the test
+	cfg = configStandalone(*cfg)
+
 	epc, err := newEtcdProcessCluster(cfg)
 	if err != nil {
 		t.Fatalf("could not start etcd process cluster (%v)", err)

e2e/etcdctl_test.go

@@ -23,19 +23,15 @@ import (
 	"github.com/coreos/etcd/pkg/testutil"
 )
 
-func TestCtlV2Set(t *testing.T)          { testCtlV2Set(t, &defaultConfig, false) }
-func TestCtlV2SetClientTLS(t *testing.T) { testCtlV2Set(t, &defaultConfigClientTLS, false) }
-func TestCtlV2SetPeerTLS(t *testing.T)   { testCtlV2Set(t, &defaultConfigPeerTLS, false) }
-func TestCtlV2SetTLS(t *testing.T)       { testCtlV2Set(t, &defaultConfigTLS, false) }
-func testCtlV2Set(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) {
+func TestCtlV2Set(t *testing.T)          { testCtlV2Set(t, &configNoTLS, false) }
+func TestCtlV2SetQuorum(t *testing.T)    { testCtlV2Set(t, &configNoTLS, true) }
+func TestCtlV2SetClientTLS(t *testing.T) { testCtlV2Set(t, &configClientTLS, false) }
+func TestCtlV2SetPeerTLS(t *testing.T)   { testCtlV2Set(t, &configPeerTLS, false) }
+func TestCtlV2SetTLS(t *testing.T)       { testCtlV2Set(t, &configTLS, false) }
+func testCtlV2Set(t *testing.T, cfg *etcdProcessClusterConfig, quorum bool) {
 	defer testutil.AfterTest(t)
 
-	mustEtcdctl(t)
-
-	epc, errC := newEtcdProcessCluster(cfg)
-	if errC != nil {
-		t.Fatalf("could not start etcd process cluster (%v)", errC)
-	}
+	epc := setupEtcdctlTest(t, cfg, quorum)
 	defer func() {
 		if errC := epc.Close(); errC != nil {
 			t.Fatalf("error closing etcd processes (%v)", errC)
@@ -44,26 +40,22 @@ func testCtlV2Set(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) {
 
 	key, value := "foo", "bar"
 
-	if err := etcdctlSet(epc, key, value, noSync); err != nil {
+	if err := etcdctlSet(epc, key, value); err != nil {
 		t.Fatalf("failed set (%v)", err)
 	}
 
-	if err := etcdctlGet(epc, key, value, noSync); err != nil {
+	if err := etcdctlGet(epc, key, value, quorum); err != nil {
 		t.Fatalf("failed get (%v)", err)
 	}
 }
 
-func TestCtlV2Mk(t *testing.T)    { testCtlV2Mk(t, &defaultConfig, false) }
-func TestCtlV2MkTLS(t *testing.T) { testCtlV2Mk(t, &defaultConfigTLS, false) }
-func testCtlV2Mk(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) {
+func TestCtlV2Mk(t *testing.T)       { testCtlV2Mk(t, &configNoTLS, false) }
+func TestCtlV2MkQuorum(t *testing.T) { testCtlV2Mk(t, &configNoTLS, true) }
+func TestCtlV2MkTLS(t *testing.T)    { testCtlV2Mk(t, &configTLS, false) }
+func testCtlV2Mk(t *testing.T, cfg *etcdProcessClusterConfig, quorum bool) {
 	defer testutil.AfterTest(t)
 
-	mustEtcdctl(t)
-
-	epc, errC := newEtcdProcessCluster(cfg)
-	if errC != nil {
-		t.Fatalf("could not start etcd process cluster (%v)", errC)
-	}
+	epc := setupEtcdctlTest(t, cfg, quorum)
 	defer func() {
 		if errC := epc.Close(); errC != nil {
 			t.Fatalf("error closing etcd processes (%v)", errC)
@@ -72,29 +64,24 @@ func testCtlV2Mk(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) {
 
 	key, value := "foo", "bar"
 
-	if err := etcdctlMk(epc, key, value, true, noSync); err != nil {
+	if err := etcdctlMk(epc, key, value, true); err != nil {
 		t.Fatalf("failed mk (%v)", err)
 	}
-	if err := etcdctlMk(epc, key, value, false, noSync); err != nil {
+	if err := etcdctlMk(epc, key, value, false); err != nil {
 		t.Fatalf("failed mk (%v)", err)
 	}
 
-	if err := etcdctlGet(epc, key, value, noSync); err != nil {
+	if err := etcdctlGet(epc, key, value, quorum); err != nil {
 		t.Fatalf("failed get (%v)", err)
 	}
 }
 
-func TestCtlV2Rm(t *testing.T)    { testCtlV2Rm(t, &defaultConfig, false) }
-func TestCtlV2RmTLS(t *testing.T) { testCtlV2Rm(t, &defaultConfigTLS, false) }
-func testCtlV2Rm(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) {
+func TestCtlV2Rm(t *testing.T)    { testCtlV2Rm(t, &configNoTLS) }
+func TestCtlV2RmTLS(t *testing.T) { testCtlV2Rm(t, &configTLS) }
+func testCtlV2Rm(t *testing.T, cfg *etcdProcessClusterConfig) {
 	defer testutil.AfterTest(t)
 
-	mustEtcdctl(t)
-
-	epc, errC := newEtcdProcessCluster(cfg)
-	if errC != nil {
-		t.Fatalf("could not start etcd process cluster (%v)", errC)
-	}
+	epc := setupEtcdctlTest(t, cfg, true)
 	defer func() {
 		if errC := epc.Close(); errC != nil {
 			t.Fatalf("error closing etcd processes (%v)", errC)
@@ -103,29 +90,25 @@ func testCtlV2Rm(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) {
 
 	key, value := "foo", "bar"
 
-	if err := etcdctlSet(epc, key, value, noSync); err != nil {
+	if err := etcdctlSet(epc, key, value); err != nil {
 		t.Fatalf("failed set (%v)", err)
 	}
 
-	if err := etcdctlRm(epc, key, value, true, noSync); err != nil {
+	if err := etcdctlRm(epc, key, value, true); err != nil {
 		t.Fatalf("failed rm (%v)", err)
 	}
-	if err := etcdctlRm(epc, key, value, false, noSync); err != nil {
+	if err := etcdctlRm(epc, key, value, false); err != nil {
 		t.Fatalf("failed rm (%v)", err)
 	}
 }
 
-func TestCtlV2Ls(t *testing.T)    { testCtlV2Ls(t, &defaultConfig, false) }
-func TestCtlV2LsTLS(t *testing.T) { testCtlV2Ls(t, &defaultConfigTLS, false) }
-func testCtlV2Ls(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) {
+func TestCtlV2Ls(t *testing.T)       { testCtlV2Ls(t, &configNoTLS, false) }
+func TestCtlV2LsQuorum(t *testing.T) { testCtlV2Ls(t, &configNoTLS, true) }
+func TestCtlV2LsTLS(t *testing.T)    { testCtlV2Ls(t, &configTLS, false) }
+func testCtlV2Ls(t *testing.T, cfg *etcdProcessClusterConfig, quorum bool) {
 	defer testutil.AfterTest(t)
 
-	mustEtcdctl(t)
-
-	epc, errC := newEtcdProcessCluster(cfg)
-	if errC != nil {
-		t.Fatalf("could not start etcd process cluster (%v)", errC)
-	}
+	epc := setupEtcdctlTest(t, cfg, quorum)
 	defer func() {
 		if errC := epc.Close(); errC != nil {
 			t.Fatalf("error closing etcd processes (%v)", errC)
@@ -134,28 +117,23 @@ func testCtlV2Ls(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) {
 
 	key, value := "foo", "bar"
 
-	if err := etcdctlSet(epc, key, value, noSync); err != nil {
+	if err := etcdctlSet(epc, key, value); err != nil {
 		t.Fatalf("failed set (%v)", err)
 	}
 
-	if err := etcdctlLs(epc, key, noSync); err != nil {
+	if err := etcdctlLs(epc, key, quorum); err != nil {
 		t.Fatalf("failed ls (%v)", err)
 	}
 }
 
-func TestCtlV2Watch(t *testing.T)                { testCtlV2Watch(t, &defaultConfig, false) }
-func TestCtlV2WatchTLS(t *testing.T)             { testCtlV2Watch(t, &defaultConfigTLS, false) }
-func TestCtlV2WatchWithProxy(t *testing.T)       { testCtlV2Watch(t, &defaultConfigWithProxy, false) }
-func TestCtlV2WatchWithProxyNoSync(t *testing.T) { testCtlV2Watch(t, &defaultConfigWithProxy, true) }
+func TestCtlV2Watch(t *testing.T)                { testCtlV2Watch(t, &configNoTLS, false) }
+func TestCtlV2WatchTLS(t *testing.T)             { testCtlV2Watch(t, &configTLS, false) }
+func TestCtlV2WatchWithProxy(t *testing.T)       { testCtlV2Watch(t, &configWithProxy, false) }
+func TestCtlV2WatchWithProxyNoSync(t *testing.T) { testCtlV2Watch(t, &configWithProxy, true) }
 func testCtlV2Watch(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) {
 	defer testutil.AfterTest(t)
 
-	mustEtcdctl(t)
-
-	epc, errC := newEtcdProcessCluster(cfg)
-	if errC != nil {
-		t.Fatalf("could not start etcd process cluster (%v)", errC)
-	}
+	epc := setupEtcdctlTest(t, cfg, true)
 	defer func() {
 		if errC := epc.Close(); errC != nil {
 			t.Fatalf("error closing etcd processes (%v)", errC)
@@ -164,7 +142,7 @@ func testCtlV2Watch(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) {
 
 	key, value := "foo", "bar"
 	errc := etcdctlWatch(epc, key, value, noSync)
-	if err := etcdctlSet(epc, key, value, noSync); err != nil {
+	if err := etcdctlSet(epc, key, value); err != nil {
 		t.Fatalf("failed set (%v)", err)
 	}
 
@@ -178,18 +156,13 @@ func testCtlV2Watch(t *testing.T, cfg *etcdProcessClusterConfig, noSync bool) {
 	}
 }
 
-func TestCtlV2GetRoleUser(t *testing.T)          { testCtlV2GetRoleUser(t, &defaultConfig) }
-func TestCtlV2GetRoleUserWithProxy(t *testing.T) { testCtlV2GetRoleUser(t, &defaultConfigWithProxy) }
+func TestCtlV2GetRoleUser(t *testing.T)          { testCtlV2GetRoleUser(t, &configNoTLS) }
+func TestCtlV2GetRoleUserWithProxy(t *testing.T) { testCtlV2GetRoleUser(t, &configWithProxy) }
 
 func testCtlV2GetRoleUser(t *testing.T, cfg *etcdProcessClusterConfig) {
 	defer testutil.AfterTest(t)
 
-	mustEtcdctl(t)
-
-	epc, cerr := newEtcdProcessCluster(cfg)
-	if cerr != nil {
-		t.Fatalf("could not start etcd process cluster (%v)", cerr)
-	}
+	epc := setupEtcdctlTest(t, cfg, true)
 	defer func() {
 		if err := epc.Close(); err != nil {
 			t.Fatalf("error closing etcd processes (%v)", err)
@@ -217,12 +190,7 @@ func testCtlV2GetRoleUser(t *testing.T, cfg *etcdProcessClusterConfig) {
 func TestCtlV2UserList(t *testing.T) {
 	defer testutil.AfterTest(t)
 
-	mustEtcdctl(t)
-
-	epc, cerr := newEtcdProcessCluster(&defaultConfigWithProxy)
-	if cerr != nil {
-		t.Fatalf("could not start etcd process cluster (%v)", cerr)
-	}
+	epc := setupEtcdctlTest(t, &configWithProxy, false)
 	defer func() {
 		if err := epc.Close(); err != nil {
 			t.Fatalf("error closing etcd processes (%v)", err)
@@ -240,12 +208,7 @@ func TestCtlV2UserList(t *testing.T) {
 func TestCtlV2RoleList(t *testing.T) {
 	defer testutil.AfterTest(t)
 
-	mustEtcdctl(t)
-
-	epc, cerr := newEtcdProcessCluster(&defaultConfigWithProxy)
-	if cerr != nil {
-		t.Fatalf("could not start etcd process cluster (%v)", cerr)
-	}
+	epc := setupEtcdctlTest(t, &configWithProxy, false)
 	defer func() {
 		if err := epc.Close(); err != nil {
 			t.Fatalf("error closing etcd processes (%v)", err)
@@ -260,7 +223,7 @@ func TestCtlV2RoleList(t *testing.T) {
 	}
 }
 
-func etcdctlPrefixArgs(clus *etcdProcessCluster, noSync bool) []string {
+func etcdctlPrefixArgs(clus *etcdProcessCluster) []string {
 	endpoints := ""
 	if proxies := clus.proxies(); len(proxies) != 0 {
 		endpoints = proxies[0].cfg.acurl.String()
@@ -272,48 +235,54 @@ func etcdctlPrefixArgs(clus *etcdProcessCluster, noSync bool) []string {
 		endpoints = strings.Join(es, ",")
 	}
 	cmdArgs := []string{"../bin/etcdctl", "--endpoints", endpoints}
-	if noSync {
-		cmdArgs = append(cmdArgs, "--no-sync")
-	}
 	if clus.cfg.isClientTLS {
 		cmdArgs = append(cmdArgs, "--ca-file", caPath, "--cert-file", certPath, "--key-file", privateKeyPath)
 	}
 	return cmdArgs
 }
 
-func etcdctlSet(clus *etcdProcessCluster, key, value string, noSync bool) error {
-	cmdArgs := append(etcdctlPrefixArgs(clus, noSync), "set", key, value)
-	return spawnWithExpect(cmdArgs, value)
+func etcdctlSet(clus *etcdProcessCluster, key, value string) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "set", key, value)
+	return spawnWithExpectedString(cmdArgs, value)
 }
 
-func etcdctlMk(clus *etcdProcessCluster, key, value string, first, noSync bool) error {
-	cmdArgs := append(etcdctlPrefixArgs(clus, noSync), "mk", key, value)
+func etcdctlMk(clus *etcdProcessCluster, key, value string, first bool) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "mk", key, value)
 	if first {
 		return spawnWithExpectedString(cmdArgs, value)
 	}
 	return spawnWithExpectedString(cmdArgs, "Error:  105: Key already exists")
 }
 
-func etcdctlGet(clus *etcdProcessCluster, key, value string, noSync bool) error {
-	cmdArgs := append(etcdctlPrefixArgs(clus, noSync), "get", key)
+func etcdctlGet(clus *etcdProcessCluster, key, value string, quorum bool) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "get", key)
+	if quorum {
+		cmdArgs = append(cmdArgs, "--quorum")
+	}
 	return spawnWithExpectedString(cmdArgs, value)
 }
 
-func etcdctlRm(clus *etcdProcessCluster, key, value string, first, noSync bool) error {
-	cmdArgs := append(etcdctlPrefixArgs(clus, noSync), "rm", key)
+func etcdctlRm(clus *etcdProcessCluster, key, value string, first bool) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "rm", key)
 	if first {
 		return spawnWithExpectedString(cmdArgs, "PrevNode.Value: "+value)
 	}
 	return spawnWithExpect(cmdArgs, "Error:  100: Key not found")
 }
 
-func etcdctlLs(clus *etcdProcessCluster, key string, noSync bool) error {
-	cmdArgs := append(etcdctlPrefixArgs(clus, noSync), "ls")
+func etcdctlLs(clus *etcdProcessCluster, key string, quorum bool) error {
+	cmdArgs := append(etcdctlPrefixArgs(clus), "ls")
+	if quorum {
+		cmdArgs = append(cmdArgs, "--quorum")
+	}
 	return spawnWithExpect(cmdArgs, key)
 }
 
 func etcdctlWatch(clus *etcdProcessCluster, key, value string, noSync bool) <-chan error {
-	cmdArgs := append(etcdctlPrefixArgs(clus, noSync), "watch", "--after-index 1", key)
+	cmdArgs := append(etcdctlPrefixArgs(clus), "watch", "--after-index 1", key)
+	if noSync {
+		cmdArgs = append(cmdArgs, "--no-sync")
+	}
 	errc := make(chan error, 1)
 	go func() {
 		errc <- spawnWithExpect(cmdArgs, value)
@@ -322,32 +291,32 @@ func etcdctlWatch(clus *etcdProcessCluster, key, value string, noSync bool) <-ch
 }
 
 func etcdctlRoleAdd(clus *etcdProcessCluster, role string) error {
-	cmdArgs := append(etcdctlPrefixArgs(clus, false), "role", "add", role)
+	cmdArgs := append(etcdctlPrefixArgs(clus), "role", "add", role)
 	return spawnWithExpectedString(cmdArgs, role)
 }
 
 func etcdctlRoleList(clus *etcdProcessCluster, expectedRole string) error {
-	cmdArgs := append(etcdctlPrefixArgs(clus, false), "role", "list")
+	cmdArgs := append(etcdctlPrefixArgs(clus), "role", "list")
 	return spawnWithExpectedString(cmdArgs, expectedRole)
 }
 
 func etcdctlUserAdd(clus *etcdProcessCluster, user, pass string) error {
-	cmdArgs := append(etcdctlPrefixArgs(clus, false), "user", "add", user+":"+pass)
+	cmdArgs := append(etcdctlPrefixArgs(clus), "user", "add", user+":"+pass)
 	return spawnWithExpectedString(cmdArgs, "User "+user+" created")
 }
 
 func etcdctlUserGrant(clus *etcdProcessCluster, user, role string) error {
-	cmdArgs := append(etcdctlPrefixArgs(clus, false), "user", "grant", "--roles", role, user)
+	cmdArgs := append(etcdctlPrefixArgs(clus), "user", "grant", "--roles", role, user)
 	return spawnWithExpectedString(cmdArgs, "User "+user+" updated")
 }
 
 func etcdctlUserGet(clus *etcdProcessCluster, user string) error {
-	cmdArgs := append(etcdctlPrefixArgs(clus, false), "user", "get", user)
+	cmdArgs := append(etcdctlPrefixArgs(clus), "user", "get", user)
 	return spawnWithExpectedString(cmdArgs, "User: "+user)
 }
 
 func etcdctlUserList(clus *etcdProcessCluster, expectedUser string) error {
-	cmdArgs := append(etcdctlPrefixArgs(clus, false), "user", "list")
+	cmdArgs := append(etcdctlPrefixArgs(clus), "user", "list")
 	return spawnWithExpectedString(cmdArgs, expectedUser)
 }
 
@@ -356,3 +325,15 @@ func mustEtcdctl(t *testing.T) {
 		t.Fatalf("could not find etcdctl binary")
 	}
 }
+
+func setupEtcdctlTest(t *testing.T, cfg *etcdProcessClusterConfig, quorum bool) *etcdProcessCluster {
+	mustEtcdctl(t)
+	if !quorum {
+		cfg = configStandalone(*cfg)
+	}
+	epc, err := newEtcdProcessCluster(cfg)
+	if err != nil {
+		t.Fatalf("could not start etcd process cluster (%v)", err)
+	}
+	return epc
+}