e2e: more debugging output for lock and elect etcdctl tests

Meant to debug #6464 and #6934

Dumps the output from the etcd/etcdctl servers and SIGQUITs to get a
golang backtrace in case of a hanged process.

e2e/ctl_v3_elect_test.go

@@ -23,9 +23,19 @@ import (
 	"github.com/coreos/etcd/pkg/expect"
 )
 
-func TestCtlV3Elect(t *testing.T) { testCtl(t, testElect) }
+func TestCtlV3Elect(t *testing.T) {
+	oldenv := os.Getenv("EXPECT_DEBUG")
+	defer os.Setenv("EXPECT_DEBUG", oldenv)
+	os.Setenv("EXPECT_DEBUG", "1")
+
+	testCtl(t, testElect)
+}
 
 func testElect(cx ctlCtx) {
+	// debugging for #6934
+	sig := cx.epc.withStopSignal(debugLockSignal)
+	defer cx.epc.withStopSignal(sig)
+
 	name := "a"
 
 	holder, ch, err := ctlV3Elect(cx, name, "p1")
@@ -102,6 +112,7 @@ func ctlV3Elect(cx ctlCtx, name, proposal string) (*expect.ExpectProcess, <-chan
 		close(outc)
 		return proc, outc, err
 	}
+	proc.StopSignal = debugLockSignal
 	go func() {
 		s, xerr := proc.ExpectFunc(func(string) bool { return true })
 		if xerr != nil {

e2e/ctl_v3_lock_test.go

@@ -16,16 +16,49 @@ package e2e
 
 import (
 	"os"
+	"runtime"
 	"strings"
+	"syscall"
 	"testing"
 	"time"
 
 	"github.com/coreos/etcd/pkg/expect"
 )
 
-func TestCtlV3Lock(t *testing.T) { testCtl(t, testLock) }
+// debugLockSignal forces SIGQUIT to debug etcdctl elect and lock failures
+var debugLockSignal os.Signal
+
+func init() {
+	// hacks to ignore SIGQUIT debugging for some builds
+	switch {
+	case os.Getenv("COVERDIR") != "":
+		// SIGQUIT interferes with coverage collection
+		debugLockSignal = syscall.SIGTERM
+	case runtime.GOARCH == "ppc64le":
+		// ppc64le's signal handling won't kill processes with SIGQUIT
+		// in the same way as amd64/i386, so processes won't terminate
+		// as expected. Since this debugging code for CI, just ignore
+		// ppc64le.
+		debugLockSignal = syscall.SIGKILL
+	default:
+		// stack dumping OK
+		debugLockSignal = syscall.SIGQUIT
+	}
+}
+
+func TestCtlV3Lock(t *testing.T) {
+	oldenv := os.Getenv("EXPECT_DEBUG")
+	defer os.Setenv("EXPECT_DEBUG", oldenv)
+	os.Setenv("EXPECT_DEBUG", "1")
+
+	testCtl(t, testLock)
+}
 
 func testLock(cx ctlCtx) {
+	// debugging for #6464
+	sig := cx.epc.withStopSignal(debugLockSignal)
+	defer cx.epc.withStopSignal(sig)
+
 	name := "a"
 
 	holder, ch, err := ctlV3Lock(cx, name)
@@ -102,6 +135,7 @@ func ctlV3Lock(cx ctlCtx, name string) (*expect.ExpectProcess, <-chan string, er
 		close(outc)
 		return proc, outc, err
 	}
+	proc.StopSignal = debugLockSignal
 	go func() {
 		s, xerr := proc.ExpectFunc(func(string) bool { return true })
 		if xerr != nil {

e2e/etcd_test.go

@@ -553,3 +553,11 @@ func (epc *etcdProcessCluster) grpcEndpoints() []string {
 	}
 	return eps
 }
+
+func (epc *etcdProcessCluster) withStopSignal(sig os.Signal) os.Signal {
+	ret := epc.procs[0].proc.StopSignal
+	for _, p := range epc.procs {
+		p.proc.StopSignal = sig
+	}
+	return ret
+}