Merge pull request #7881 from heyitsanthony/testctl-logging

e2e: more debugging output for lock and elect tests

e2e/ctl_v3_elect_test.go

@@ -23,9 +23,19 @@ import (
 	"github.com/coreos/etcd/pkg/expect"
 )
 
-func TestCtlV3Elect(t *testing.T) { testCtl(t, testElect) }
+func TestCtlV3Elect(t *testing.T) {
+	oldenv := os.Getenv("EXPECT_DEBUG")
+	defer os.Setenv("EXPECT_DEBUG", oldenv)
+	os.Setenv("EXPECT_DEBUG", "1")
+
+	testCtl(t, testElect)
+}
 
 func testElect(cx ctlCtx) {
+	// debugging for #6934
+	sig := cx.epc.withStopSignal(debugLockSignal)
+	defer cx.epc.withStopSignal(sig)
+
 	name := "a"
 
 	holder, ch, err := ctlV3Elect(cx, name, "p1")
@@ -102,6 +112,7 @@ func ctlV3Elect(cx ctlCtx, name, proposal string) (*expect.ExpectProcess, <-chan
 		close(outc)
 		return proc, outc, err
 	}
+	proc.StopSignal = debugLockSignal
 	go func() {
 		s, xerr := proc.ExpectFunc(func(string) bool { return true })
 		if xerr != nil {

e2e/ctl_v3_lock_test.go

@@ -16,16 +16,49 @@ package e2e
 
 import (
 	"os"
+	"runtime"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 
 	"github.com/coreos/etcd/pkg/expect"
 )
 
-func TestCtlV3Lock(t *testing.T) { testCtl(t, testLock) }
+// debugLockSignal forces SIGQUIT to debug etcdctl elect and lock failures
+var debugLockSignal os.Signal
+
+func init() {
+	// hacks to ignore SIGQUIT debugging for some builds
+	switch {
+	case os.Getenv("COVERDIR") != "":
+		// SIGQUIT interferes with coverage collection
+		debugLockSignal = syscall.SIGTERM
+	case runtime.GOARCH == "ppc64le":
+		// ppc64le's signal handling won't kill processes with SIGQUIT
+		// in the same way as amd64/i386, so processes won't terminate
+		// as expected. Since this debugging code for CI, just ignore
+		// ppc64le.
+		debugLockSignal = syscall.SIGKILL
+	default:
+		// stack dumping OK
+		debugLockSignal = syscall.SIGQUIT
+	}
+}
+
+func TestCtlV3Lock(t *testing.T) {
+	oldenv := os.Getenv("EXPECT_DEBUG")
+	defer os.Setenv("EXPECT_DEBUG", oldenv)
+	os.Setenv("EXPECT_DEBUG", "1")
+
+	testCtl(t, testLock)
+}
 
 func testLock(cx ctlCtx) {
+	// debugging for #6464
+	sig := cx.epc.withStopSignal(debugLockSignal)
+	defer cx.epc.withStopSignal(sig)
+
 	name := "a"
 
 	holder, ch, err := ctlV3Lock(cx, name)
@@ -102,6 +135,7 @@ func ctlV3Lock(cx ctlCtx, name string) (*expect.ExpectProcess, <-chan string, er
 		close(outc)
 		return proc, outc, err
 	}
+	proc.StopSignal = debugLockSignal
 	go func() {
 		s, xerr := proc.ExpectFunc(func(string) bool { return true })
 		if xerr != nil {

e2e/etcd_test.go

@@ -553,3 +553,11 @@ func (epc *etcdProcessCluster) grpcEndpoints() []string {
 	}
 	return eps
 }
+
+func (epc *etcdProcessCluster) withStopSignal(sig os.Signal) os.Signal {
+	ret := epc.procs[0].proc.StopSignal
+	for _, p := range epc.procs {
+		p.proc.StopSignal = sig
+	}
+	return ret
+}

integration/cluster_test.go

@@ -447,7 +447,9 @@ func TestRejectUnhealthyRemove(t *testing.T) {
 // (see https://github.com/coreos/etcd/issues/7512 for more).
 func TestRestartRemoved(t *testing.T) {
 	defer testutil.AfterTest(t)
+
 	capnslog.SetGlobalLogLevel(capnslog.INFO)
+	defer capnslog.SetGlobalLogLevel(defaultLogLevel)
 
 	// 1. start single-member cluster
 	c := NewCluster(t, 1)

integration/logger_test.go

@@ -16,6 +16,8 @@ package integration
 
 import "github.com/coreos/pkg/capnslog"
 
+const defaultLogLevel = capnslog.CRITICAL
+
 func init() {
-	capnslog.SetGlobalLogLevel(capnslog.CRITICAL)
+	capnslog.SetGlobalLogLevel(defaultLogLevel)
 }

pkg/expect/expect.go

@@ -44,8 +44,6 @@ type ExpectProcess struct {
 	StopSignal os.Signal
 }
 
-var printDebugLines = os.Getenv("EXPECT_DEBUG") != ""
-
 // NewExpect creates a new process for expect testing.
 func NewExpect(name string, arg ...string) (ep *ExpectProcess, err error) {
 	// if env[] is nil, use current system env
@@ -75,6 +73,7 @@ func NewExpectWithEnv(name string, args []string, env []string) (ep *ExpectProce
 
 func (ep *ExpectProcess) read() {
 	defer ep.wg.Done()
+	printDebugLines := os.Getenv("EXPECT_DEBUG") != ""
 	r := bufio.NewReader(ep.fpty)
 	for ep.err == nil {
 		ep.ptyMu.Lock()