Merge pull request #8128 from gyuho/functional-tester

*: run basic functional-tester cases to test script

test

@@ -16,6 +16,11 @@ set -e
 
 source ./build
 
+# build before setting up test GOPATH
+if [[ "${PASSES}" == *"functional"* ]]; then
+	./tools/functional-tester/build
+fi
+
 # build tests with vendored dependencies
 etcd_setup_gopath
 
@@ -88,6 +93,35 @@ function integration_pass {
 	go test -timeout 1m -v ${RACE} -cpu 1,2,4 -run=Example $@ ${TEST}
 }
 
+function functional_pass {
+	for a in 1 2 3; do
+		mkdir -p ./agent-$a
+		./bin/etcd-agent -etcd-path ./bin/etcd -etcd-log-dir "./agent-$a" -port ":${a}9027" -use-root=false &
+		pid="$!"
+		agent_pids="${agent_pids} $pid"
+	done
+
+	./bin/etcd-tester \
+		-agent-endpoints "127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027" \
+		-client-ports 12379,22379,32379 \
+		-peer-ports 12380,22380,32380 \
+		-limit 1 \
+		-schedule-cases "0 1 2 3 4 5" \
+		-exit-on-failure
+	ETCD_TESTER_EXIT_CODE=$?
+	echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE}
+
+	echo "Waiting for processes to exit"
+	kill -s TERM ${agent_pids}
+	for a in ${agent_pids}; do wait $a || true; done
+	rm -rf ./agent-*
+
+	if [[ "${ETCD_TESTER_EXIT_CODE}" -ne "0" ]]; then
+		echo "FAIL with exit code" ${ETCD_TESTER_EXIT_CODE}
+		exit ${ETCD_TESTER_EXIT_CODE}
+	fi
+}
+
 function cov_pass {
 	echo "Running code coverage..."
 	# install gocovmerge before running code coverage from github.com/wadey/gocovmerge

tools/functional-tester/etcd-tester/main.go

@@ -46,6 +46,7 @@ func main() {
 	stressKeySize := flag.Uint("stress-key-size", 100, "the size of each small key written into etcd.")
 	stressKeySuffixRange := flag.Uint("stress-key-count", 250000, "the count of key range written into etcd.")
 	limit := flag.Int("limit", -1, "the limit of rounds to run failure set (-1 to run without limits).")
+	exitOnFailure := flag.Bool("exit-on-failure", false, "exit tester on first failure")
 	stressQPS := flag.Int("stress-qps", 10000, "maximum number of stresser requests per second.")
 	schedCases := flag.String("schedule-cases", "", "test case schedule")
 	consistencyCheck := flag.Bool("consistency-check", true, "true to check consistency (revision, hash)")
@@ -125,9 +126,10 @@ func main() {
 	}
 
 	t := &tester{
-		failures: schedule,
-		cluster:  c,
-		limit:    *limit,
+		failures:      schedule,
+		cluster:       c,
+		limit:         *limit,
+		exitOnFailure: *exitOnFailure,
 
 		scfg:         scfg,
 		stresserType: *stresserType,

tools/functional-tester/etcd-tester/tester.go

@@ -16,12 +16,14 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"time"
 )
 
 type tester struct {
-	cluster *cluster
-	limit   int
+	cluster       *cluster
+	limit         int
+	exitOnFailure bool
 
 	failures        []failure
 	status          Status
@@ -49,6 +51,7 @@ func (tt *tester) runLoop() {
 
 	if err := tt.resetStressCheck(); err != nil {
 		plog.Errorf("%s failed to start stresser (%v)", tt.logPrefix(), err)
+		tt.failed()
 		return
 	}
 
@@ -87,6 +90,7 @@ func (tt *tester) runLoop() {
 		if round > 0 && round%500 == 0 { // every 500 rounds
 			if err := tt.defrag(); err != nil {
 				plog.Warningf("%s functional-tester returning with error (%v)", tt.logPrefix(), err)
+				tt.failed()
 				return
 			}
 		}
@@ -209,7 +213,18 @@ func (tt *tester) logPrefix() string {
 	return prefix
 }
 
+func (tt *tester) failed() {
+	if !tt.exitOnFailure {
+		return
+	}
+	plog.Warningf("%s exiting on failure", tt.logPrefix())
+	tt.cluster.Terminate()
+	os.Exit(2)
+}
+
 func (tt *tester) cleanup() error {
+	defer tt.failed()
+
 	roundFailedTotalCounter.Inc()
 	desc := "compact/defrag"
 	if tt.status.Case != -1 {