etcd-tester: add 'exit-on-failure'

Signed-off-by: Gyu-Ho Lee <gyuhox@gmail.com>

tools/functional-tester/etcd-tester/main.go

@@ -46,6 +46,7 @@ func main() {
 	stressKeySize := flag.Uint("stress-key-size", 100, "the size of each small key written into etcd.")
 	stressKeySuffixRange := flag.Uint("stress-key-count", 250000, "the count of key range written into etcd.")
 	limit := flag.Int("limit", -1, "the limit of rounds to run failure set (-1 to run without limits).")
+	exitOnFailure := flag.Bool("exit-on-failure", false, "exit tester on first failure")
 	stressQPS := flag.Int("stress-qps", 10000, "maximum number of stresser requests per second.")
 	schedCases := flag.String("schedule-cases", "", "test case schedule")
 	consistencyCheck := flag.Bool("consistency-check", true, "true to check consistency (revision, hash)")
@@ -125,9 +126,10 @@ func main() {
 	}
 
 	t := &tester{
-		failures: schedule,
-		cluster:  c,
-		limit:    *limit,
+		failures:      schedule,
+		cluster:       c,
+		limit:         *limit,
+		exitOnFailure: *exitOnFailure,
 
 		scfg:         scfg,
 		stresserType: *stresserType,

tools/functional-tester/etcd-tester/tester.go

@@ -16,12 +16,14 @@ package main
 
 import (
 	"fmt"
+	"os"
 	"time"
 )
 
 type tester struct {
-	cluster *cluster
-	limit   int
+	cluster       *cluster
+	limit         int
+	exitOnFailure bool
 
 	failures        []failure
 	status          Status
@@ -49,6 +51,7 @@ func (tt *tester) runLoop() {
 
 	if err := tt.resetStressCheck(); err != nil {
 		plog.Errorf("%s failed to start stresser (%v)", tt.logPrefix(), err)
+		tt.failed()
 		return
 	}
 
@@ -87,6 +90,7 @@ func (tt *tester) runLoop() {
 		if round > 0 && round%500 == 0 { // every 500 rounds
 			if err := tt.defrag(); err != nil {
 				plog.Warningf("%s functional-tester returning with error (%v)", tt.logPrefix(), err)
+				tt.failed()
 				return
 			}
 		}
@@ -209,7 +213,18 @@ func (tt *tester) logPrefix() string {
 	return prefix
 }
 
+func (tt *tester) failed() {
+	if !tt.exitOnFailure {
+		return
+	}
+	plog.Warningf("%s exiting on failure", tt.logPrefix())
+	tt.cluster.Terminate()
+	os.Exit(2)
+}
+
 func (tt *tester) cleanup() error {
+	defer tt.failed()
+
 	roundFailedTotalCounter.Inc()
 	desc := "compact/defrag"
 	if tt.status.Case != -1 {