functional-tester: a new option -failure-wrapper for enabling/disabling external fault injector

This commit adds a new option -failure-wrapper to etcd-tester. The
option receives a path of script that is used for enabling/disabling
external fault injectors. The script is called with an option "enable"
when it needs to be enabled (when failure.Inject() is called) and
called with "disabled" in an opposite case (when failure.Recover() is
called).

tools/functional-tester/etcd-tester/failure.go

@@ -17,6 +17,7 @@ package main
 import (
 	"fmt"
 	"math/rand"
+	"os/exec"
 	"time"
 )
 
@@ -177,3 +178,20 @@ type failureNop failureByFunc
 
 func (f *failureNop) Inject(c *cluster, round int) error  { return nil }
 func (f *failureNop) Recover(c *cluster, round int) error { return nil }
+
+type failureExternal struct {
+	failure
+
+	description string
+	scriptPath  string
+}
+
+func (f *failureExternal) Inject(c *cluster, round int) error {
+	return exec.Command(f.scriptPath, "enable", fmt.Sprintf("%d", round)).Run()
+}
+
+func (f *failureExternal) Recover(c *cluster, round int) error {
+	return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", round)).Run()
+}
+
+func (f *failureExternal) Desc() string { return f.description }

tools/functional-tester/etcd-tester/failure_agent.go

@@ -145,3 +145,10 @@ func newFailureNop() failure {
 		description: "no failure",
 	}
 }
+
+func newFailureExternal(scriptPath string) failure {
+	return &failureExternal{
+		description: fmt.Sprintf("external fault injector (script: %s)", scriptPath),
+		scriptPath:  scriptPath,
+	}
+}

tools/functional-tester/etcd-tester/main.go

@@ -50,6 +50,7 @@ func main() {
 	isV2Only := flag.Bool("v2-only", false, "'true' to run V2 only tester.")
 	stresserType := flag.String("stresser", "default", "specify stresser (\"default\" or \"nop\").")
 	failureTypes := flag.String("failures", "default,failpoints", "specify failures (concat of \"default\" and \"failpoints\").")
+	externalFailures := flag.String("external-failures", "", "specify a path of script for enabling/disabling an external fault injector")
 	flag.Parse()
 
 	eps := strings.Split(*endpointStr, ",")
@@ -93,6 +94,14 @@ func main() {
 		failures = makeFailures(*failureTypes, c)
 	}
 
+	if externalFailures != nil && *externalFailures != "" {
+		if len(failures) != 0 {
+			plog.Errorf("specify only one of -failures or -external-failures")
+			os.Exit(1)
+		}
+		failures = append(failures, newFailureExternal(*externalFailures))
+	}
+
 	if len(failures) == 0 {
 		plog.Infof("no failures\n")
 		failures = append(failures, newFailureNop())