Merge pull request #7937 from heyitsanthony/e2e-close-timeout

e2e: Stop() lock/elect etcdctl process if Close times out

e2e/ctl_v3_elect_test.go

@@ -80,7 +80,7 @@ func testElect(cx ctlCtx) {
 	if err = blocked.Signal(os.Interrupt); err != nil {
 		cx.t.Fatal(err)
 	}
-	if err = blocked.Close(); err != nil {
+	if err := closeWithTimeout(blocked, time.Second); err != nil {
 		cx.t.Fatal(err)
 	}
 
@@ -88,7 +88,7 @@ func testElect(cx ctlCtx) {
 	if err = holder.Signal(os.Interrupt); err != nil {
 		cx.t.Fatal(err)
 	}
-	if err = holder.Close(); err != nil {
+	if err = closeWithTimeout(holder, time.Second); err != nil {
 		cx.t.Fatal(err)
 	}
 

e2e/ctl_v3_lock_test.go

@@ -103,7 +103,7 @@ func testLock(cx ctlCtx) {
 	if err = blocked.Signal(os.Interrupt); err != nil {
 		cx.t.Fatal(err)
 	}
-	if err = blocked.Close(); err != nil {
+	if err = closeWithTimeout(blocked, time.Second); err != nil {
 		cx.t.Fatal(err)
 	}
 
@@ -111,7 +111,7 @@ func testLock(cx ctlCtx) {
 	if err = holder.Signal(os.Interrupt); err != nil {
 		cx.t.Fatal(err)
 	}
-	if err = holder.Close(); err != nil {
+	if err = closeWithTimeout(holder, time.Second); err != nil {
 		cx.t.Fatal(err)
 	}
 

e2e/etcd_test.go

@@ -20,6 +20,7 @@ import (
 	"net/url"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/coreos/etcd/etcdserver"
 	"github.com/coreos/etcd/pkg/expect"
@@ -561,3 +562,17 @@ func (epc *etcdProcessCluster) withStopSignal(sig os.Signal) os.Signal {
 	}
 	return ret
 }
+
+func closeWithTimeout(p *expect.ExpectProcess, d time.Duration) error {
+	errc := make(chan error, 1)
+	go func() { errc <- p.Close() }()
+	select {
+	case err := <-errc:
+		return err
+	case <-time.After(d):
+		p.Stop()
+		// retry close after stopping to collect SIGQUIT data, if any
+		closeWithTimeout(p, time.Second)
+	}
+	return fmt.Errorf("took longer than %v to Close process %+v", d, p)
+}