Merge pull request #7939 from gyuho/test

etcd-tester: add '-failpoints' to configure gofail

tools/functional-tester/etcd-tester/failpoint.go

@@ -32,7 +32,7 @@ type failpointStats struct {
 
 var fpStats failpointStats
 
-func failpointFailures(c *cluster) (ret []failure, err error) {
+func failpointFailures(c *cluster, failpoints []string) (ret []failure, err error) {
 	var fps []string
 	fps, err = failpointPaths(c.Members[0].FailpointURL)
 	if err != nil {
@@ -43,7 +43,7 @@ func failpointFailures(c *cluster) (ret []failure, err error) {
 		if len(fp) == 0 {
 			continue
 		}
-		fpFails := failuresFromFailpoint(fp)
+		fpFails := failuresFromFailpoint(fp, failpoints)
 		// wrap in delays so failpoint has time to trigger
 		for i, fpf := range fpFails {
 			if strings.Contains(fp, "Snap") {
@@ -77,34 +77,39 @@ func failpointPaths(endpoint string) ([]string, error) {
 	return fps, nil
 }
 
-func failuresFromFailpoint(fp string) []failure {
-	inject := makeInjectFailpoint(fp, `panic("etcd-tester")`)
+// failpoints follows FreeBSD KFAIL_POINT syntax.
+// e.g. panic("etcd-tester"),1*sleep(1000)->panic("etcd-tester")
+func failuresFromFailpoint(fp string, failpoints []string) (fs []failure) {
 	recov := makeRecoverFailpoint(fp)
-	return []failure{
-		&failureOne{
-			description:   description("failpoint " + fp + " panic one"),
-			injectMember:  inject,
-			recoverMember: recov,
-		},
-		&failureAll{
-			description:   description("failpoint " + fp + " panic all"),
-			injectMember:  inject,
-			recoverMember: recov,
-		},
-		&failureMajority{
-			description:   description("failpoint " + fp + " panic majority"),
-			injectMember:  inject,
-			recoverMember: recov,
-		},
-		&failureLeader{
-			failureByFunc{
-				description:   description("failpoint " + fp + " panic leader"),
+	for _, failpoint := range failpoints {
+		inject := makeInjectFailpoint(fp, failpoint)
+		fs = append(fs, []failure{
+			&failureOne{
+				description:   description(fmt.Sprintf("failpoint %s (one: %s)", fp, failpoint)),
 				injectMember:  inject,
 				recoverMember: recov,
 			},
-			0,
-		},
+			&failureAll{
+				description:   description(fmt.Sprintf("failpoint %s (all: %s)", fp, failpoint)),
+				injectMember:  inject,
+				recoverMember: recov,
+			},
+			&failureMajority{
+				description:   description(fmt.Sprintf("failpoint %s (majority: %s)", fp, failpoint)),
+				injectMember:  inject,
+				recoverMember: recov,
+			},
+			&failureLeader{
+				failureByFunc{
+					description:   description(fmt.Sprintf("failpoint %s (leader: %s)", fp, failpoint)),
+					injectMember:  inject,
+					recoverMember: recov,
+				},
+				0,
+			},
+		}...)
 	}
+	return fs
 }
 
 func makeInjectFailpoint(fp, val string) injectMemberFunc {

tools/functional-tester/etcd-tester/main.go

@@ -52,6 +52,7 @@ func main() {
 	stresserType := flag.String("stresser", "keys,lease", "comma separated list of stressers (keys, lease, v2keys, nop, election-runner, watch-runner, lock-racer-runner, lease-runner).")
 	etcdRunnerPath := flag.String("etcd-runner", "", "specify a path of etcd runner binary")
 	failureTypes := flag.String("failures", "default,failpoints", "specify failures (concat of \"default\" and \"failpoints\").")
+	failpoints := flag.String("failpoints", `panic("etcd-tester")`, `comma separated list of failpoint terms to inject (e.g. 'panic("etcd-tester"),1*sleep(1000)')`)
 	externalFailures := flag.String("external-failures", "", "specify a path of script for enabling/disabling an external fault injector")
 	enablePprof := flag.Bool("enable-pprof", false, "true to enable pprof")
 	flag.Parse()
@@ -81,7 +82,8 @@ func main() {
 	var failures []failure
 
 	if failureTypes != nil && *failureTypes != "" {
-		failures = makeFailures(*failureTypes, c)
+		types, failpoints := strings.Split(*failureTypes, ","), strings.Split(*failpoints, ",")
+		failures = makeFailures(types, failpoints, c)
 	}
 
 	if externalFailures != nil && *externalFailures != "" {
@@ -170,12 +172,10 @@ func portsFromArg(arg string, n, defaultPort int) []int {
 	return ret
 }
 
-func makeFailures(types string, c *cluster) []failure {
+func makeFailures(types, failpoints []string, c *cluster) []failure {
 	var failures []failure
-
-	fails := strings.Split(types, ",")
-	for i := range fails {
-		switch fails[i] {
+	for i := range types {
+		switch types[i] {
 		case "default":
 			defaultFailures := []failure{
 				newFailureKillAll(),
@@ -193,14 +193,14 @@ func makeFailures(types string, c *cluster) []failure {
 			failures = append(failures, defaultFailures...)
 
 		case "failpoints":
-			fpFailures, fperr := failpointFailures(c)
+			fpFailures, fperr := failpointFailures(c, failpoints)
 			if len(fpFailures) == 0 {
 				plog.Infof("no failpoints found (%v)", fperr)
 			}
 			failures = append(failures, fpFailures...)
 
 		default:
-			plog.Errorf("unknown failure: %s\n", fails[i])
+			plog.Errorf("unknown failure: %s\n", types[i])
 			os.Exit(1)
 		}
 	}