Merge pull request #7254 from fanminshi/rework_coverage_e2e

e2e: add code coverage to e2e

e2e/etcd_spawn_cov.go.go

@@ -0,0 +1,82 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build cov
+
+package e2e
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/coreos/etcd/pkg/expect"
+	"github.com/coreos/etcd/pkg/fileutil"
+	"github.com/coreos/etcd/pkg/flags"
+)
+
+func spawnCmd(args []string) (*expect.ExpectProcess, error) {
+	if args[0] == binPath {
+		coverPath := os.Getenv("COVERDIR")
+		if !filepath.IsAbs(coverPath) {
+			// COVERDIR is relative to etcd root but e2e test has its path set to be relative to the e2e folder.
+			// adding ".." in front of COVERDIR ensures that e2e saves coverage reports to the correct location.
+			coverPath = filepath.Join("..", coverPath)
+		}
+		if !fileutil.Exist(coverPath) {
+			return nil, fmt.Errorf("could not find coverage folder")
+		}
+		covArgs := []string{
+			fmt.Sprintf("-test.coverprofile=e2e.%v.coverprofile", time.Now().UnixNano()),
+			"-test.outputdir=" + coverPath,
+		}
+		ep := expect.NewExpectWithEnv(binDir+"/etcd_test", covArgs, args2env(args[1:]))
+		// ep sends SIGTERM to etcd_test process on ep.close()
+		// allowing the process to exit gracefully in order to generate a coverage report.
+		// note: go runtime ignores SIGINT but not SIGTERM
+		// if e2e test is run as a background process.
+		ep.StopSignal = syscall.SIGTERM
+		return nil, ep
+	}
+	return expect.NewExpect(args[0], args[1:]...)
+}
+
+func args2env(args []string) []string {
+	var covEnvs []string
+	for i := range args[1:] {
+		if !strings.HasPrefix(args[i], "--") {
+			continue
+		}
+		flag := strings.Split(args[i], "--")[1]
+		val := "true"
+		// split the flag that has "="
+		// e.g --auto-tls=true" => flag=auto-tls and val=true
+		if strings.Contains(args[i], "=") {
+			split := strings.Split(flag, "=")
+			flag = split[0]
+			val = split[1]
+		}
+
+		if i+1 < len(args) {
+			if !strings.HasPrefix(args[i+1], "--") {
+				val = args[i+1]
+			}
+		}
+		covEnvs = append(covEnvs, flags.FlagToEnv("ETCD", flag)+"="+val)
+	}
+	return covEnvs
+}

e2e/etcd_spawn_nocov.go

@@ -0,0 +1,23 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !cov
+
+package e2e
+
+import "github.com/coreos/etcd/pkg/expect"
+
+func spawnCmd(args []string) (*expect.ExpectProcess, error) {
+	return expect.NewExpect(args[0], args[1:]...)
+}

e2e/etcd_test.go

@@ -494,10 +494,6 @@ func waitReadyExpectProc(exproc *expect.ExpectProcess, isProxy bool) error {
 	return err
 }
 
-func spawnCmd(args []string) (*expect.ExpectProcess, error) {
-	return expect.NewExpect(args[0], args[1:]...)
-}
-
 func spawnWithExpect(args []string, expected string) error {
 	return spawnWithExpects(args, []string{expected}...)
 }
@@ -515,10 +511,10 @@ func spawnWithExpects(args []string, xs ...string) error {
 	)
 	for _, txt := range xs {
 		for {
-			l, err := proc.ExpectFunc(lineFunc)
-			if err != nil {
+			l, lerr := proc.ExpectFunc(lineFunc)
+			if lerr != nil {
 				proc.Close()
-				return fmt.Errorf("%v (expected %q, got %q)", err, txt, lines)
+				return fmt.Errorf("%v (expected %q, got %q)", lerr, txt, lines)
 			}
 			lines = append(lines, l)
 			if strings.Contains(l, txt) {
@@ -527,9 +523,6 @@ func spawnWithExpects(args []string, xs ...string) error {
 		}
 	}
 	perr := proc.Close()
-	if err != nil {
-		return err
-	}
 	if len(xs) == 0 && proc.LineCount() != 0 { // expect no output
 		return fmt.Errorf("unexpected output (got lines %q, line count %d)", lines, proc.LineCount())
 	}

etcdmain/config.go

@@ -52,6 +52,9 @@ var (
 		"snapshot",
 		"v",
 		"vv",
+		// for coverage testing
+		"test.coverprofile",
+		"test.outputdir",
 	}
 )
 

main_test.go

@@ -0,0 +1,35 @@
+// Copyright 2017 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"testing"
+)
+
+func TestMain(t *testing.T) {
+	// don't launch etcd server when invoked via go test
+	if strings.HasSuffix(os.Args[0], "etcd.test") {
+		return
+	}
+
+	notifier := make(chan os.Signal, 1)
+	signal.Notify(notifier, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL)
+	go main()
+	<-notifier
+}

pkg/expect/expect.go

@@ -23,6 +23,7 @@ import (
 	"os/exec"
 	"strings"
 	"sync"
+	"syscall"
 
 	"github.com/kr/pty"
 )
@@ -38,13 +39,27 @@ type ExpectProcess struct {
 	lines []string
 	count int // increment whenever new line gets added
 	err   error
+
+	// StopSignal is the signal Stop sends to the process; defaults to SIGKILL.
+	StopSignal os.Signal
 }
 
 var printDebugLines = os.Getenv("EXPECT_DEBUG") != ""
 
 // NewExpect creates a new process for expect testing.
 func NewExpect(name string, arg ...string) (ep *ExpectProcess, err error) {
-	ep = &ExpectProcess{cmd: exec.Command(name, arg...)}
+	// if env[] is nil, use current system env
+	return NewExpectWithEnv(name, arg, nil)
+}
+
+// NewExpectWithEnv creates a new process with user defined env variables for expect testing.
+func NewExpectWithEnv(name string, args []string, env []string) (ep *ExpectProcess, err error) {
+	cmd := exec.Command(name, args...)
+	cmd.Env = env
+	ep = &ExpectProcess{
+		cmd:        cmd,
+		StopSignal: syscall.SIGKILL,
+	}
 	ep.cond = sync.NewCond(&ep.mu)
 	ep.cmd.Stderr = ep.cmd.Stdout
 	ep.cmd.Stdin = nil
@@ -132,7 +147,7 @@ func (ep *ExpectProcess) close(kill bool) error {
 		return ep.err
 	}
 	if kill {
-		ep.cmd.Process.Kill()
+		ep.Signal(ep.StopSignal)
 	}
 
 	err := ep.cmd.Wait()

test

@@ -10,7 +10,8 @@
 # PKG=snap ./test
 #
 # Run code coverage 
-# COVERDIR=coverage PASSES=cov ./test
+# COVERDIR must either be a absolute path or a relative path to the etcd root
+# COVERDIR=coverage PASSES="build_cov cov" ./test
 set -e
 
 source ./build
@@ -94,21 +95,31 @@ function cov_pass {
 		exit 255
 	fi
 
+	if [ ! -f "bin/etcd_test" ]; then
+		echo "etcd_test binary not found"
+		exit 255
+	fi
+
 	mkdir -p "$COVERDIR"
 
 	# PKGS_DELIM contains all the core etcd pkgs delimited by ',' which will be profiled for code coverage.
 	# Integration tests will generate code coverage for those pkgs 
 	PKGS_DELIM=$(echo $TEST | sed 's/ /,/g')
 
-	# TODO create coverage to e2e test
 	PKGS=`echo "$TEST_PKGS" | egrep -v "(e2e|functional-tester)"`
-
+	# run code coverage for unit and integration tests
 	for t in ${PKGS}; do
 		tf=`echo $t | tr / _`
-		#  uses -run=Test to skip examples because clientv3/ example tests will leak goroutines
+		# uses -run=Test to skip examples because clientv3/ example tests will leak goroutines
 		go test -covermode=set -coverpkg $PKGS_DELIM -timeout 15m -run=Test -v -coverprofile "$COVERDIR/${tf}.coverprofile"  ${REPO_PATH}/$t
 	done
 
+	# run code coverage for e2e tests
+	# use 30m timeout because e2e coverage takes longer 
+	# due to many tests cause etcd process to wait
+	# on leadership transfer timeout during gracefully shutdown
+	go test -tags cov -timeout 30m -v ${REPO_PATH}"/e2e"
+
 	gocovmerge "$COVERDIR"/*.coverprofile >"$COVERDIR"/cover.out
 }
 
@@ -283,6 +294,14 @@ function dep_pass {
 	fi
 }
 
+function build_cov_pass {
+	out="bin"
+	if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
+	PKGS=$TEST
+	ETCD_PKGS_DELIM=$(echo $PKGS | sed 's/ /,/g')
+	go test -c -covermode=set -coverpkg=$ETCD_PKGS_DELIM -o ${out}/etcd_test 
+}
+
 function compile_pass {
 	echo "Checking build..."
 	go build -v ./tools/...