tests/*: clean up travis, semaphore scripts

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

.travis.yml

@@ -6,8 +6,7 @@ sudo: required
 services: docker
 
 go:
-- "1.8.7"
-- tip
+- 1.8.7
 
 notifications:
   on_success: never
@@ -15,37 +14,20 @@ notifications:
 
 env:
   matrix:
-  - TARGET=amd64
-  - TARGET=amd64-go-tip
-  - TARGET=darwin-amd64
-  - TARGET=windows-amd64
-  - TARGET=arm64
-  - TARGET=arm
-  - TARGET=386
-  - TARGET=ppc64le
+  - TARGET=linux-amd64-build
+  - TARGET=linux-amd64-unit
+  - TARGET=linux-amd64-integration
+  - TARGET=linux-amd64-functional
+  - TARGET=linux-386-build
+  - TARGET=linux-386-unit
+  - TARGET=darwin-amd64-build
+  - TARGET=windows-amd64-build
+  - TARGET=linux-arm-build
+  - TARGET=linux-arm64-build
+  - TARGET=linux-ppc64le-build
 
 matrix:
   fast_finish: true
-  allow_failures:
-  - go: tip
-    env: TARGET=amd64-go-tip
-  exclude:
-  - go: "1.8.7"
-    env: TARGET=amd64-go-tip
-  - go: tip
-    env: TARGET=amd64
-  - go: tip
-    env: TARGET=darwin-amd64
-  - go: tip
-    env: TARGET=windows-amd64
-  - go: tip
-    env: TARGET=arm
-  - go: tip
-    env: TARGET=arm64
-  - go: tip
-    env: TARGET=386
-  - go: tip
-    env: TARGET=ppc64le
 
 before_install:
 - if [[ $TRAVIS_GO_VERSION == 1.* ]]; then docker pull gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION}; fi
@@ -57,33 +39,59 @@ script:
  - echo "TRAVIS_GO_VERSION=${TRAVIS_GO_VERSION}"
  - >
     case "${TARGET}" in
-      amd64)
+      linux-amd64-build)
         docker run --rm \
           --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 ./test"
+          /bin/bash -c "GOARCH=amd64 PASSES='build' ./test"
         ;;
-      amd64-go-tip)
-        GOARCH=amd64 ./test
+      linux-amd64-unit)
+        docker run --rm \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=amd64 PASSES='unit' ./test"
+        ;;
+      linux-amd64-integration)
+        docker run --rm \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=amd64 PASSES='integration' ./test"
+        ;;
+      linux-amd64-functional)
+        docker run --rm \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "./build && GOARCH=amd64 PASSES='functional' ./test"
+        ;;
+      linux-386-build)
+        docker run --rm \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=386 PASSES='build' ./test"
+        ;;
+      linux-386-unit)
+        docker run --rm \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GOARCH=386 PASSES='unit' ./test"
+        ;;
+      darwin-amd64-build)
+        docker run --rm \
+          --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
+          /bin/bash -c "GO_BUILD_FLAGS='-v' GOOS=darwin GOARCH=amd64 ./build"
         ;;
-      darwin-amd64)
+      windows-amd64-build)
         docker run --rm \
           --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GO_BUILD_FLAGS='-a -v' GOOS=darwin GOARCH=amd64 ./build"
+          /bin/bash -c "GO_BUILD_FLAGS='-v' GOOS=windows GOARCH=amd64 ./build"
         ;;
-      windows-amd64)
+      linux-arm-build)
         docker run --rm \
           --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GO_BUILD_FLAGS='-a -v' GOOS=windows GOARCH=amd64 ./build"
+          /bin/bash -c "GO_BUILD_FLAGS='-v' GOARCH=arm ./build"
         ;;
-      386)
+      linux-arm64-build)
         docker run --rm \
           --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=386 PASSES='build unit' ./test"
+          /bin/bash -c "GO_BUILD_FLAGS='-v' GOARCH=arm64 ./build"
         ;;
-      *)
-        # test building out of gopath
+      linux-ppc64le-build)
         docker run --rm \
           --volume=`pwd`:/go/src/github.com/coreos/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GO_BUILD_FLAGS='-a -v' GOARCH='${TARGET}' ./build"
+          /bin/bash -c "GO_BUILD_FLAGS='-v' GOARCH=ppc64le ./build"
         ;;
     esac

test

@@ -4,46 +4,67 @@
 # ./test
 # ./test -v
 #
+#
+# Run specified test pass
+#
+# $ PASSES=unit ./test
+# $ PASSES=integration ./test
+#
+#
 # Run tests for one package
+# Each pass has different default timeout, if you just run tests in one package or 1 test case then you can set TIMEOUT
+# flag for different expectation
+#
+# $ PASSES=unit PKG=./wal TIMEOUT=1m ./test
+# $ PASSES=integration PKG=client/integration TIMEOUT=1m ./test
+#
+#
+# Run specified unit tests in one package
+# To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew ";
+# to run only "TestNew", set "TESTCASE="\bTestNew\b""
+#
+# $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test
+# $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test
+# $ PASSES=integration PKG=client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test
 #
-# PKG=./wal ./test
-# PKG=snap ./test
 #
 # Run code coverage
 # COVERDIR must either be a absolute path or a relative path to the etcd root
-# COVERDIR=coverage PASSES="build_cov cov" ./test
+# $ COVERDIR=coverage PASSES="build_cov cov" ./test
 set -e
 
 source ./build
 
 # build before setting up test GOPATH
 if [[ "${PASSES}" == *"functional"* ]]; then
-	./tools/functional-tester/build
+	./functional/build
 fi
 
-# build tests with vendored dependencies
-etcd_setup_gopath
-
 if [ -z "$PASSES" ]; then
-	PASSES="fmt bom dep compile build unit"
+	PASSES="fmt bom dep build unit"
 fi
 
 USERPKG=${PKG:-}
 
-# Invoke ./cover for HTML output
+# Invoke ./tests/cover.test.bash for HTML output
 COVER=${COVER:-"-cover"}
 
 # Hack: gofmt ./ will recursively check the .git directory. So use *.go for gofmt.
 IGNORE_PKGS="(cmd/|etcdserverpb|rafttest|gopath.proto|v3lockpb|v3electionpb)"
-INTEGRATION_PKGS="(integration|e2e|contrib|functional-tester)"
+INTEGRATION_PKGS="(integration|e2e|contrib|functional)"
 
 # all github.com/coreos/etcd/whatever pkgs that are not auto-generated / tools
+# shellcheck disable=SC1117
 PKGS=$(find . -name \*.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | grep -vE "(tools/|contrib/|e2e|pb)" | sed "s|\.|${REPO_PATH}|g" | xargs echo)
 # pkg1,pkg2,pkg3
 PKGS_COMMA=${PKGS// /,}
 
+# shellcheck disable=SC1117
 TEST_PKGS=$(find . -name \*_test.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g")
+
+# shellcheck disable=SC1117
 FORMATTABLE=$(find . -name \*.go | while read -r a; do echo "$(dirname "$a")/*.go"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g")
+
 TESTABLE_AND_FORMATTABLE=$(echo "$TEST_PKGS" | grep -vE "$INTEGRATION_PKGS")
 
 # check if user provided PKG override
@@ -58,17 +79,22 @@ else
 	# only run gofmt on packages provided by user
 	FMT="$TEST"
 fi
+
+# shellcheck disable=SC2206
 FMT=($FMT)
 
 # prepend REPO_PATH to each local package
 split=$TEST
 TEST=""
 for a in $split; do TEST="$TEST ${REPO_PATH}/${a}"; done
+
+# shellcheck disable=SC2206
 TEST=($TEST)
 
 # TODO: 'client' pkg fails with gosimple from generated files
 # TODO: 'rafttest' is failing with unused
 STATIC_ANALYSIS_PATHS=$(find . -name \*.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | grep -v 'client')
+# shellcheck disable=SC2206
 STATIC_ANALYSIS_PATHS=($STATIC_ANALYSIS_PATHS)
 
 if [ -z "$GOARCH" ]; then
@@ -80,6 +106,11 @@ if [ "$GOARCH" == "amd64" ]; then
 	RACE="--race"
 fi
 
+RUN_ARG=""
+if [ ! -z "${TESTCASE}" ]; then
+	RUN_ARG="-run=${TESTCASE}"
+fi
+
 function unit_pass {
 	echo "Running unit tests..."
 	GO_TEST_FLAG=""
@@ -90,25 +121,69 @@ function unit_pass {
 		GO_TEST_FLAG="-v"
 		export CLIENT_DEBUG=1
 	fi
-	# only -run=Test so examples can run in integration tests
-	go test ${GO_TEST_FLAG} -timeout 5m "${COVER}" ${RACE} -cpu 1,2,4 -run=Test "$@" "${TEST[@]}"
+
+	if [ "${RUN_ARG}" == "" ]; then
+	    RUN_ARG="-run=Test"
+	fi
+
+	# check if user provided time out, especially useful when just run one test case
+	# expectation could be different
+	USERTIMEOUT=""
+	if [ -z "${TIMEOUT}" ]; then
+		USERTIMEOUT="3m"
+	else
+		USERTIMEOUT="${TIMEOUT}"
+	fi
+	go test ${GO_TEST_FLAG} -timeout "${USERTIMEOUT}"  "${COVER}" ${RACE} -cpu 1,2,4 ${RUN_ARG} "$@" "${TEST[@]}"
 }
 
 function integration_pass {
 	echo "Running integration tests..."
-	go test -timeout 15m -v -cpu 1,2,4 $@ ${REPO_PATH}/integration
-	go test -timeout 1m -v ${RACE} -cpu 1,2,4 $@ ${REPO_PATH}/client/integration
-	go test -timeout 10m -v ${RACE} -cpu 1,2,4 $@ ${REPO_PATH}/clientv3/integration
-	go test -timeout 1m -v -cpu 1,2,4 $@ ${REPO_PATH}/contrib/raftexample
-	go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example $@ ${TEST}
+
+	# check if user provided time out, especially useful when just run one test case
+	# expectation could be different
+	USERTIMEOUT=""
+	if [ -z "${TIMEOUT}" ]; then
+		USERTIMEOUT="20m"
+	else
+		USERTIMEOUT="${TIMEOUT}"
+	fi
+
+	# if TESTCASE and PKG set, run specified test case in specified PKG
+	# if TESTCASE set, PKG not set, run specified test case in all integration and integration_extra packages
+	# if TESTCASE not set, PKG set, run all test cases in specified package
+	# if TESTCASE not set, PKG not set, run all tests in all integration and integration_extra packages
+	if [ -z "${TESTCASE}" ] && [ -z "${USERPKG}" ]; then
+		go test -timeout "${USERTIMEOUT}" -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration"
+		integration_extra "$@"
+	else
+		if [ -z "${USERPKG}" ]; then
+			INTEGTESTPKG=("${REPO_PATH}/integration"
+						  "${REPO_PATH}/client/integration"
+						  "${REPO_PATH}/clientv3/integration"
+						  "${REPO_PATH}/contrib/raftexample"
+						  "${REPO_PATH}/store")
+		else
+			INTEGTESTPKG=("${TEST[@]}")
+		fi
+		go test -timeout "${USERTIMEOUT}" -v -cpu 1,2,4 "${RUN_ARG}"  "$@" "${INTEGTESTPKG[@]}"
+	fi
+}
+
+function integration_extra {
+	go test -timeout 1m -v ${RACE} -cpu 1,2,4 "$@" "${REPO_PATH}/client/integration"
+	go test -timeout 25m -v ${RACE} -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration"
+	go test -timeout 1m -v -cpu 1,2,4 "$@" "${REPO_PATH}/contrib/raftexample"
+	go test -timeout 5m -v ${RACE} -tags v2v3 "$@" "${REPO_PATH}/etcdserver/v2store"
+	go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example "$@" "${TEST[@]}"
 }
 
 function functional_pass {
-        # Clean up any data and logs from previous runs
-  	rm -rf ./agent-*
+  	# Clean up any data and logs from previous runs
+  	rm -rf /tmp/etcd-agent-data-*
+
 	for a in 1 2 3; do
-		mkdir -p ./agent-$a
-		./bin/etcd-agent -etcd-path ./bin/etcd -etcd-log-dir "./agent-$a" -port ":${a}9027" -use-root=false &
+		./bin/etcd-agent --network tcp --address 127.0.0.1:${a}9027 &
 		pid="$!"
 		agent_pids="${agent_pids} $pid"
 	done
@@ -120,43 +195,116 @@ function functional_pass {
 		done
 	done
 
-	echo "Starting 'etcd-tester'"
-	./bin/etcd-tester \
-		-agent-endpoints "127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027" \
-		-client-ports 12379,22379,32379 \
-		-peer-ports 12380,22380,32380 \
-		-limit 1 \
-		-schedule-cases "0 1 2 3 4 5" \
-		-exit-on-failure && echo "'etcd-tester' succeeded"
+	echo "functional test START!"
+	./bin/etcd-tester --config ./functional.yaml && echo "'etcd-tester' succeeded"
 	ETCD_TESTER_EXIT_CODE=$?
 	echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE}
 
-	echo "Waiting for processes to exit"
-	kill -s TERM ${agent_pids}
-	for a in ${agent_pids}; do wait $a || true; done
+	# shellcheck disable=SC2206
+	agent_pids=($agent_pids)
+	kill -s TERM "${agent_pids[@]}" || true
 
 	if [[ "${ETCD_TESTER_EXIT_CODE}" -ne "0" ]]; then
 		echo "--- FAIL: exit code" ${ETCD_TESTER_EXIT_CODE}
 		exit ${ETCD_TESTER_EXIT_CODE}
 	fi
+	echo "functional test PASS!"
+}
+
+function cov_pass {
+	echo "Running code coverage..."
+	# install gocovmerge before running code coverage from github.com/wadey/gocovmerge
+	# gocovmerge merges coverage files
+	if ! which gocovmerge >/dev/null; then
+		echo "gocovmerge not installed"
+		exit 255
+	fi
+
+	if [ -z "$COVERDIR" ]; then
+		echo "COVERDIR undeclared"
+		exit 255
+	fi
+
+	if [ ! -f "bin/etcd_test" ]; then
+		echo "etcd_test binary not found"
+		exit 255
+	fi
+
+	mkdir -p "$COVERDIR"
+
+	# run code coverage for unit and integration tests
+	GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 20m"
+	# shellcheck disable=SC2206
+	GOCOVFLAGS=($GOCOVFLAGS)
+	failed=""
+	for t in $(echo "${TEST_PKGS}" | grep -vE "(e2e|functional)"); do
+		tf=$(echo "$t" | tr / _)
+		# cache package compilation data for faster repeated builds
+		go test "${GOCOVFLAGS[@]}" -i "${REPO_PATH}/$t" || true
+		# uses -run=Test to skip examples because clientv3/ example tests will leak goroutines
+		go test "${GOCOVFLAGS[@]}" -run=Test -coverprofile "$COVERDIR/${tf}.coverprofile"  "${REPO_PATH}/$t" || failed="$failed $t"
+	done
+
+	# v2v3 tests
+	go test -tags v2v3 "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/store-v2v3.coverprofile" "${REPO_PATH}/clientv3/integration" || failed="$failed store-v2v3"
+
+	# proxy tests
+	go test -tags cluster_proxy "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/proxy_integration.coverprofile" "${REPO_PATH}/integration" || failed="$failed proxy-integration"
+	go test -tags cluster_proxy "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/proxy_clientv3.coverprofile" "${REPO_PATH}/clientv3/integration" || failed="$failed proxy-clientv3/integration"
+
+	# run code coverage for e2e tests
+	# use 30m timeout because e2e coverage takes longer
+	# due to many tests cause etcd process to wait
+	# on leadership transfer timeout during gracefully shutdown
+	echo Testing e2e without proxy...
+	go test -tags cov -timeout 30m -v "${REPO_PATH}/e2e" || failed="$failed e2e"
+	echo Testing e2e with proxy...
+	go test -tags "cov cluster_proxy" -timeout 30m -v "${REPO_PATH}/e2e" || failed="$failed e2e-proxy"
+
+	# incrementally merge to get coverage data even if some coverage files are corrupted
+	# optimistically assume etcdserver package's coverage file is OK since gocovmerge
+	# expects to start with a non-empty file
+	cp "$COVERDIR"/etcdserver.coverprofile "$COVERDIR"/cover.out
+	for f in "$COVERDIR"/*.coverprofile; do
+		echo "merging test coverage file ${f}"
+		gocovmerge "$f" "$COVERDIR"/cover.out  >"$COVERDIR"/cover.tmp || failed="$failed $f"
+		if [ -s "$COVERDIR"/cover.tmp ]; then
+			mv "$COVERDIR"/cover.tmp "$COVERDIR"/cover.out
+		fi
+	done
+	# strip out generated files (using GNU-style sed)
+	sed --in-place '/generated.go/d' "$COVERDIR"/cover.out || true
+
+	# held failures to generate the full coverage file, now fail
+	if [ -n "$failed" ]; then
+		for f in $failed; do
+			echo "--- FAIL:" "$f"
+		done
+		exit 255
+	fi
 }
 
 function e2e_pass {
 	echo "Running e2e tests..."
-	go test -timeout 15m -v -cpu 1,2,4 $@ ${REPO_PATH}/e2e
-}
 
-function integration_extra {
-	go test -timeout 15m -v ${RACE} -cpu 1,2,4 "$@" "${REPO_PATH}/client/integration"
-	go test -timeout 20m -v ${RACE} -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration"
+	# check if user provided time out, especially useful when just run one test case
+	# expectation could be different
+	USERTIMEOUT=""
+	if [ -z "${TIMEOUT}" ]; then
+		USERTIMEOUT="20m"
+	else
+		USERTIMEOUT="${TIMEOUT}"
+	fi
+
+	go test -timeout "${USERTIMEOUT}" -v -cpu 1,2,4 "${RUN_ARG}"  "$@" "${REPO_PATH}/e2e"
 }
 
 function integration_e2e_pass {
 	echo "Running integration and e2e tests..."
 
-	go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" &
+	go test -timeout 20m -v -cpu 1,2,4 "$@" "${REPO_PATH}/e2e" &
 	e2epid="$!"
-	go test -timeout 15m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" &
+	go test -timeout 20m -v -cpu 1,2,4 "$@" "${REPO_PATH}/integration" &
 	intpid="$!"
 	wait $e2epid
 	wait $intpid
@@ -164,20 +312,21 @@ function integration_e2e_pass {
 }
 
 function grpcproxy_pass {
-	go test -timeout 15m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 $@ ${REPO_PATH}/integration
-	go test -timeout 15m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 $@ ${REPO_PATH}/clientv3/integration
+	go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/integration"
+	go test -timeout 20m -v ${RACE} -tags cluster_proxy -cpu 1,2,4 "$@" "${REPO_PATH}/clientv3/integration"
+	go test -timeout 20m -v -tags cluster_proxy "$@" "${REPO_PATH}/e2e"
 }
 
 function release_pass {
 	rm -f ./bin/etcd-last-release
 	# to grab latest patch release; bump this up for every minor release
-	UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.2.*" | head -1)
+	UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1)
 	if [ -n "$MANUAL_VER" ]; then
 		# in case, we need to test against different version
 		UPGRADE_VER=$MANUAL_VER
 	fi
 	if [[ -z ${UPGRADE_VER} ]]; then
-		UPGRADE_VER="v3.2.0"
+		UPGRADE_VER="v3.3.0"
 		echo "fallback to" ${UPGRADE_VER}
 	fi
 
@@ -185,7 +334,7 @@ function release_pass {
 	echo "Downloading $file"
 
 	set +e
-	curl --fail -L https://github.com/coreos/etcd/releases/download/$UPGRADE_VER/$file -o /tmp/$file
+	curl --fail -L "https://github.com/coreos/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
 	local result=$?
 	set -e
 	case $result in
@@ -195,96 +344,167 @@ function release_pass {
 			;;
 	esac
 
-	tar xzvf /tmp/$file -C /tmp/ --strip-components=1
+	tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
 	mkdir -p ./bin
 	mv /tmp/etcd ./bin/etcd-last-release
 }
 
-function fmt_pass {
-	toggle_failpoints disable
-
-	echo "Checking gofmt..."
-	fmtRes=$(gofmt -l -s -d $FMT)
-	if [ -n "${fmtRes}" ]; then
-		echo -e "gofmt checking failed:\n${fmtRes}"
-		exit 255
-	fi
-
-	echo "Checking govet..."
-	vetRes=$(go vet $TEST)
-	if [ -n "${vetRes}" ]; then
-		echo -e "govet checking failed:\n${vetRes}"
-		exit 255
+function shellcheck_pass {
+	if which shellcheck >/dev/null; then
+		shellcheckResult=$(shellcheck -fgcc build test scripts/*.sh 2>&1 || true)
+		if [ -n "${shellcheckResult}" ]; then
+			echo -e "shellcheck checking failed:\\n${shellcheckResult}"
+			exit 255
+		fi
 	fi
+}
 
-	echo "Checking documentation style..."
+function markdown_you_pass {
 	# eschew you
-	yous=`find . -name \*.md -exec egrep --color "[Yy]ou[r]?[ '.,;]" {} + | grep -v /v2/ || true`
+	yous=$(find . -name \*.md -exec grep -E --color "[Yy]ou[r]?[ '.,;]" {} + | grep -v /v2/ || true)
 	if [ ! -z "$yous" ]; then
-		echo -e "found 'you' in documentation:\n${yous}"
+		echo -e "found 'you' in documentation:\\n${yous}"
 		exit 255
 	fi
+}
 
+function markdown_marker_pass {
 	# TODO: check other markdown files when marker handles headers with '[]'
 	if which marker >/dev/null; then
-		echo "Checking marker to find broken links..."
-		markerResult=`marker --skip-http --root ./Documentation 2>&1 || true`
+		markerResult=$(marker --skip-http --root ./Documentation 2>&1 || true)
 		if [ -n "${markerResult}" ]; then
-			echo -e "marker checking failed:\n${markerResult}"
+			echo -e "marker checking failed:\\n${markerResult}"
 			exit 255
 		fi
 	else
 		echo "Skipping marker..."
 	fi
+}
+
+function goword_pass {
+	if which goword >/dev/null; then
+		# get all go files to process
+		gofiles=$(find "${FMT[@]}" -iname '*.go' 2>/dev/null)
+		# shellcheck disable=SC2206
+		gofiles_all=($gofiles)
+		# ignore tests and protobuf files
+		# shellcheck disable=SC1117
+		gofiles=$(echo "${gofiles_all[@]}" | sort | uniq | sed "s/ /\n/g" | grep -vE "(\\_test.go|\\.pb\\.go)")
+		# shellcheck disable=SC2206
+		gofiles=($gofiles)
+		# only check for broken exported godocs
+		gowordRes=$(goword -use-spell=false "${gofiles[@]}" | grep godoc-export | sort)
+		if [ ! -z "$gowordRes" ]; then
+			echo -e "goword checking failed:\\n${gowordRes}"
+			exit 255
+		fi
+		# check some spelling
+		gowordRes=$(goword -ignore-file=.words clientv3/{*,*/*}.go 2>&1 | grep spell | sort)
+		if [ ! -z "$gowordRes" ]; then
+			echo -e "goword checking failed:\\n${gowordRes}"
+			exit 255
+		fi
+	else
+		echo "Skipping goword..."
+	fi
+}
+
+function gofmt_pass {
+	fmtRes=$(gofmt -l -s -d "${FMT[@]}")
+	if [ -n "${fmtRes}" ]; then
+		echo -e "gofmt checking failed:\\n${fmtRes}"
+		exit 255
+	fi
+}
+
+function govet_pass {
+	vetRes=$(go vet "${TEST[@]}")
+	if [ -n "${vetRes}" ]; then
+		echo -e "govet checking failed:\\n${vetRes}"
+		exit 255
+	fi
+}
+
+function govet_shadow_pass {
+	fmtpkgs=$(for a in "${FMT[@]}"; do dirname "$a"; done | sort | uniq | grep -v "\\.")
+	# shellcheck disable=SC2206
+	fmtpkgs=($fmtpkgs)
+	vetRes=$(go tool vet -all -shadow "${fmtpkgs[@]}" 2>&1 | grep -v '/gw/' || true)
+	if [ -n "${vetRes}" ]; then
+		echo -e "govet -all -shadow checking failed:\\n${vetRes}"
+		exit 255
+	fi
+}
 
+function gosimple_pass {
 	if which gosimple >/dev/null; then
-		echo "Checking gosimple..."
-		gosimpleResult=`gosimple ${STATIC_ANALYSIS_PATHS} 2>&1 || true`
+		gosimpleResult=$(gosimple "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
 		if [ -n "${gosimpleResult}" ]; then
-			# TODO: resolve these after go1.8 migration
-			SIMPLE_CHECK_MASK="S(1024)"
-			if echo "${gosimpleResult}" | egrep -v "$SIMPLE_CHECK_MASK"; then
-				echo -e "gosimple checking failed:\n${gosimpleResult}"
-				exit 255
-			else
-				echo -e "gosimple warning:\n${gosimpleResult}"
-			fi
+			echo -e "gosimple checking failed:\\n${gosimpleResult}"
+			exit 255
 		fi
 	else
 		echo "Skipping gosimple..."
 	fi
+}
 
+function unused_pass {
 	if which unused >/dev/null; then
-		echo "Checking unused..."
-		unusedResult=`unused ${STATIC_ANALYSIS_PATHS} 2>&1 || true`
+		unusedResult=$(unused "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
 		if [ -n "${unusedResult}" ]; then
-			echo -e "unused checking failed:\n${unusedResult}"
+			echo -e "unused checking failed:\\n${unusedResult}"
 			exit 255
 		fi
 	else
 		echo "Skipping unused..."
 	fi
+}
 
+function staticcheck_pass {
 	if which staticcheck >/dev/null; then
-		echo "Checking staticcheck..."
-		staticcheckResult=`staticcheck ${STATIC_ANALYSIS_PATHS} 2>&1 || true`
+		staticcheckResult=$(staticcheck "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
 		if [ -n "${staticcheckResult}" ]; then
 			# TODO: resolve these after go1.8 migration
 			# See https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck
-			STATIC_CHECK_MASK="SA(1019|2002)"
-			if echo "${staticcheckResult}" | egrep -v "$STATIC_CHECK_MASK"; then
-				echo -e "staticcheck checking failed:\n${staticcheckResult}"
+			STATIC_CHECK_MASK="SA(1012|1019|2002)"
+			if echo "${staticcheckResult}" | grep -vE "$STATIC_CHECK_MASK"; then
+				echo -e "staticcheck checking failed:\\n${staticcheckResult}"
 				exit 255
 			else
-				suppressed=`echo "${staticcheckResult}" | sed 's/ /\n/g' | grep "(SA" | sort | uniq -c`
-				echo -e "staticcheck suppressed warnings:\n${suppressed}"
+				suppressed=$(echo "${staticcheckResult}" | sed 's/ /\n/g' | grep "(SA" | sort | uniq -c)
+				echo -e "staticcheck suppressed warnings:\\n${suppressed}"
 			fi
 		fi
 	else
 		echo "Skipping staticcheck..."
 	fi
+}
+
+function ineffassign_pass {
+	if which ineffassign >/dev/null; then
+		ineffassignResult=$(ineffassign "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
+		if [ -n "${ineffassignResult}" ]; then
+			echo -e "ineffassign checking failed:\\n${ineffassignResult}"
+			exit 255
+		fi
+	else
+		echo "Skipping ineffassign..."
+	fi
+}
+
+function nakedret_pass {
+	if which nakedret >/dev/null; then
+		nakedretResult=$(nakedret "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true)
+		if [ -n "${nakedretResult}" ]; then
+			echo -e "nakedret checking failed:\\n${nakedretResult}"
+			exit 255
+		fi
+	else
+		echo "Skipping nakedret..."
+	fi
+}
 
-	echo "Checking for license header..."
+function license_header_pass {
 	licRes=""
 	files=$(find . -type f -iname '*.go' ! -path './cmd/*' ! -path './gopath.proto/*')
 	for file in $files; do
@@ -293,13 +513,30 @@ function fmt_pass {
 		fi
 	done
 	if [ -n "${licRes}" ]; then
-		echo -e "license header checking failed:\n${licRes}"
+		echo -e "license header checking failed:\\n${licRes}"
 		exit 255
 	fi
+}
+
+function receiver_name_pass {
+	# shellcheck disable=SC1117
+	recvs=$(grep 'func ([^*]' {*,*/*,*/*/*}.go  | grep -Ev "(generated|pb/)" | tr  ':' ' ' |  \
+		awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" |  sort  | uniq  | \
+		grep -Ev  "(Descriptor|Proto|_)"  | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ')
+	if [ -n "${recvs}" ]; then
+		# shellcheck disable=SC2206
+		recvs=($recvs)
+		for recv in "${recvs[@]}"; do
+			echo "Mismatched receiver for $recv..."
+			grep "$recv" "${FMT[@]}" | grep 'func ('
+		done
+		exit 255
+	fi
+}
 
-	echo "Checking commit titles..."
-	git log --oneline "$(git merge-base HEAD master)"...HEAD | while read l; do
-		commitMsg=`echo "$l" | cut -f2- -d' '`
+function commit_title_pass {
+	git log --oneline "$(git merge-base HEAD master)"...HEAD | while read -r l; do
+		commitMsg=$(echo "$l" | cut -f2- -d' ')
 		if [[ "$commitMsg" == Merge* ]]; then
 			# ignore "Merge pull" commits
 			continue
@@ -309,11 +546,11 @@ function fmt_pass {
 			continue
 		fi
 
-		pkgPrefix=`echo "$commitMsg" | cut -f1 -d':'`
-		spaceCommas=`echo "$commitMsg" | sed 's/ /\n/g' | grep -c ',$' || echo 0`
-		commaSpaces=`echo "$commitMsg" | sed 's/,/\n/g' | grep -c '^ ' || echo 0`
-		if [[ `echo $commitMsg | grep -c ":..*"` == 0 || "$commitMsg" == "$pkgPrefix" || "$spaceCommas" != "$commaSpaces" ]]; then
-    			echo "$l"...
+		pkgPrefix=$(echo "$commitMsg" | cut -f1 -d':')
+		spaceCommas=$(echo "$commitMsg" | sed 's/ /\n/g' | grep -c ',$' || echo 0)
+		commaSpaces=$(echo "$commitMsg" | sed 's/,/\n/g' | grep -c '^ ' || echo 0)
+		if [[ $(echo "$commitMsg" | grep -c ":..*") == 0 || "$commitMsg" == "$pkgPrefix" || "$spaceCommas" != "$commaSpaces" ]]; then
+			echo "$l"...
 			echo "Expected commit title format '<package>{\", \"<package>}: <description>'"
 			echo "Got: $l"
 			exit 255
@@ -321,6 +558,31 @@ function fmt_pass {
 	done
 }
 
+function fmt_pass {
+	toggle_failpoints disable
+
+	for p in shellcheck \
+			markdown_you \
+			markdown_marker \
+			goword \
+			gofmt \
+			govet \
+			govet_shadow \
+			gosimple \
+			unused \
+			staticcheck \
+			ineffassign \
+			nakedret \
+			license_header \
+			receiver_name \
+			commit_title \
+			; do
+		echo "'$p' started at $(date)"
+		"${p}"_pass "$@"
+		echo "'$p' completed at $(date)"
+	done
+}
+
 function bom_pass {
 	if ! which license-bill-of-materials >/dev/null; then
 		return
@@ -341,10 +603,10 @@ function dep_pass {
 	# don't pull in etcdserver package
 	pushd clientv3 >/dev/null
 	badpkg="(etcdserver$|mvcc$|backend$|grpc-gateway)"
-	deps=`go list -f '{{ .Deps }}'  | sed 's/ /\n/g' | egrep "${badpkg}" || echo ""`
+	deps=$(go list -f '{{ .Deps }}'  | sed 's/ /\n/g' | grep -E "${badpkg}" || echo "")
 	popd >/dev/null
 	if [ ! -z "$deps" ]; then
-		echo -e "clientv3 has masked dependencies:\n${deps}"
+		echo -e "clientv3 has masked dependencies:\\n${deps}"
 		exit 255
 	fi
 }
@@ -352,22 +614,21 @@ function dep_pass {
 function build_cov_pass {
 	out="bin"
 	if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi
-	go test -tags cov -c -covermode=set -coverpkg=$PKGS_COMMA -o ${out}/etcd_test
-	go test -tags cov -c -covermode=set -coverpkg=$PKGS_COMMA -o ${out}/etcdctl_test ${REPO_PATH}/etcdctl
-}
-
-function compile_pass {
-	echo "Checking build..."
-	go build -v ./tools/...
+	go test -tags cov -c -covermode=set -coverpkg="$PKGS_COMMA" -o "${out}/etcd_test"
+	go test -tags cov -c -covermode=set -coverpkg="$PKGS_COMMA" -o "${out}/etcdctl_test" "${REPO_PATH}/etcdctl"
 }
 
 # fail fast on static tests
 function build_pass {
+	echo "Checking build..."
 	GO_BUILD_FLAGS="-a -v" etcd_build
+	GO_BUILD_FLAGS="-a -v" tools_build
 }
 
 for pass in $PASSES; do
-	${pass}_pass $@
+	echo "Starting '$pass' pass at $(date)"
+	"${pass}"_pass "$@"
+	echo "Finished '$pass' pass at $(date)"
 done
 
 echo "Success"

tests/semaphore.test.bash

@@ -7,15 +7,15 @@ fi
 
 TEST_SUFFIX=$(date +%s | base64 | head -c 15)
 
-TEST_OPTS="PASSES='build unit release integration_e2e functional' MANUAL_VER=v3.2.17"
+TEST_OPTS="PASSES='build release e2e' MANUAL_VER=v3.2.17"
 if [ "$TEST_ARCH" == "386" ]; then
-	TEST_OPTS="GOARCH=386 PASSES='build unit integration_e2e'"
+  TEST_OPTS="GOARCH=386 PASSES='build e2e'"
 fi
 
 docker run \
-	--rm \
-	--volume=`pwd`:/go/src/github.com/coreos/etcd \
-	gcr.io/etcd-development/etcd-test:go1.8.7 \
-	/bin/bash -c "${TEST_OPTS} ./test 2>&1 | tee test-${TEST_SUFFIX}.log"
+  --rm \
+  --volume=`pwd`:/go/src/github.com/coreos/etcd \
+  gcr.io/etcd-development/etcd-test:go1.8.7 \
+  /bin/bash -c "${TEST_OPTS} ./test 2>&1 | tee test-${TEST_SUFFIX}.log"
 
 ! egrep "(--- FAIL:|panic: test timed out|appears to have leaked)" -B50 -A10 test-${TEST_SUFFIX}.log