functional-tester: improve logging, remove "round" parameter from

"Failure"

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

tools/functional-tester/tester/cluster_tester.go

@@ -34,9 +34,9 @@ func (clus *Cluster) StartTester() {
 	var preModifiedKey int64
 	for round := 0; round < int(clus.Tester.RoundLimit) || clus.Tester.RoundLimit == -1; round++ {
 		roundTotalCounter.Inc()
-
 		clus.rd = round
-		if err := clus.doRound(round); err != nil {
+
+		if err := clus.doRound(); err != nil {
 			clus.logger.Warn(
 				"doRound failed; returning",
 				zap.Int("round", clus.rd),
@@ -50,6 +50,7 @@ func (clus *Cluster) StartTester() {
 			preModifiedKey = 0
 			continue
 		}
+
 		// -1 so that logPrefix doesn't print out 'case'
 		clus.cs = -1
 
@@ -105,11 +106,16 @@ func (clus *Cluster) StartTester() {
 	)
 }
 
-func (clus *Cluster) doRound(round int) error {
+func (clus *Cluster) doRound() error {
 	if clus.Tester.FailureShuffle {
 		clus.shuffleFailures()
 	}
 
+	clus.logger.Info(
+		"starting round",
+		zap.Int("round", clus.rd),
+		zap.Strings("failures", clus.failureStrings()),
+	)
 	for i, f := range clus.failures {
 		clus.cs = i
 
@@ -126,7 +132,7 @@ func (clus *Cluster) doRound(round int) error {
 			zap.Int("case", clus.cs),
 			zap.String("desc", f.Desc()),
 		)
-		if err := f.Inject(clus, round); err != nil {
+		if err := f.Inject(clus); err != nil {
 			return fmt.Errorf("injection error: %v", err)
 		}
 		clus.logger.Info(
@@ -145,7 +151,7 @@ func (clus *Cluster) doRound(round int) error {
 			zap.Int("case", clus.cs),
 			zap.String("desc", f.Desc()),
 		)
-		if err := f.Recover(clus, round); err != nil {
+		if err := f.Recover(clus); err != nil {
 			return fmt.Errorf("recovery error: %v", err)
 		}
 		clus.logger.Info(
@@ -174,6 +180,12 @@ func (clus *Cluster) doRound(round int) error {
 			zap.String("desc", f.Desc()),
 		)
 	}
+
+	clus.logger.Info(
+		"finished round",
+		zap.Int("round", clus.rd),
+		zap.Strings("failures", clus.failureStrings()),
+	)
 	return nil
 }
 

tools/functional-tester/tester/failure.go

@@ -21,10 +21,10 @@ package tester
 type Failure interface {
 	// Inject injeccts the failure into the testing cluster at the given
 	// round. When calling the function, the cluster should be in health.
-	Inject(clus *Cluster, round int) error
+	Inject(clus *Cluster) error
 	// Recover recovers the injected failure caused by the injection of the
 	// given round and wait for the recovery of the testing cluster.
-	Recover(clus *Cluster, round int) error
+	Recover(clus *Cluster) error
 	// Desc returns a description of the failure
 	Desc() string
 }

tools/functional-tester/tester/failure_case_delay.go

@@ -25,8 +25,8 @@ type failureDelay struct {
 	delayDuration time.Duration
 }
 
-func (f *failureDelay) Inject(clus *Cluster, round int) error {
-	if err := f.Failure.Inject(clus, round); err != nil {
+func (f *failureDelay) Inject(clus *Cluster) error {
+	if err := f.Failure.Inject(clus); err != nil {
 		return err
 	}
 	if f.delayDuration > 0 {

tools/functional-tester/tester/failure_case_external.go

@@ -26,12 +26,12 @@ type failureExternal struct {
 	scriptPath  string
 }
 
-func (f *failureExternal) Inject(clus *Cluster, round int) error {
-	return exec.Command(f.scriptPath, "enable", fmt.Sprintf("%d", round)).Run()
+func (f *failureExternal) Inject(clus *Cluster) error {
+	return exec.Command(f.scriptPath, "enable", fmt.Sprintf("%d", clus.rd)).Run()
 }
 
-func (f *failureExternal) Recover(clus *Cluster, round int) error {
-	return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", round)).Run()
+func (f *failureExternal) Recover(clus *Cluster) error {
+	return exec.Command(f.scriptPath, "disable", fmt.Sprintf("%d", clus.rd)).Run()
 }
 
 func (f *failureExternal) Desc() string { return f.description }

tools/functional-tester/tester/failure_case_kill.go

@@ -99,19 +99,19 @@ type failureLeader struct {
 // failureUntilSnapshot injects a failure and waits for a snapshot event
 type failureUntilSnapshot struct{ Failure }
 
-func (f *failureOne) Inject(clus *Cluster, round int) error {
-	return f.injectMember(clus, round%len(clus.Members))
+func (f *failureOne) Inject(clus *Cluster) error {
+	return f.injectMember(clus, clus.rd%len(clus.Members))
 }
 
-func (f *failureOne) Recover(clus *Cluster, round int) error {
-	if err := f.recoverMember(clus, round%len(clus.Members)); err != nil {
+func (f *failureOne) Recover(clus *Cluster) error {
+	if err := f.recoverMember(clus, clus.rd%len(clus.Members)); err != nil {
 		return err
 	}
 	clus.logger.Info("wait health after recovering failureOne")
 	return clus.WaitHealth()
 }
 
-func (f *failureAll) Inject(clus *Cluster, round int) error {
+func (f *failureAll) Inject(clus *Cluster) error {
 	for i := range clus.Members {
 		if err := f.injectMember(clus, i); err != nil {
 			return err
@@ -120,7 +120,7 @@ func (f *failureAll) Inject(clus *Cluster, round int) error {
 	return nil
 }
 
-func (f *failureAll) Recover(clus *Cluster, round int) error {
+func (f *failureAll) Recover(clus *Cluster) error {
 	for i := range clus.Members {
 		if err := f.recoverMember(clus, i); err != nil {
 			return err
@@ -130,8 +130,8 @@ func (f *failureAll) Recover(clus *Cluster, round int) error {
 	return clus.WaitHealth()
 }
 
-func (f *failureQuorum) Inject(clus *Cluster, round int) error {
-	for i := range killMap(len(clus.Members), round) {
+func (f *failureQuorum) Inject(clus *Cluster) error {
+	for i := range killMap(len(clus.Members), clus.rd) {
 		if err := f.injectMember(clus, i); err != nil {
 			return err
 		}
@@ -139,8 +139,8 @@ func (f *failureQuorum) Inject(clus *Cluster, round int) error {
 	return nil
 }
 
-func (f *failureQuorum) Recover(clus *Cluster, round int) error {
-	for i := range killMap(len(clus.Members), round) {
+func (f *failureQuorum) Recover(clus *Cluster) error {
+	for i := range killMap(len(clus.Members), clus.rd) {
 		if err := f.recoverMember(clus, i); err != nil {
 			return err
 		}
@@ -148,7 +148,7 @@ func (f *failureQuorum) Recover(clus *Cluster, round int) error {
 	return nil
 }
 
-func (f *failureLeader) Inject(clus *Cluster, round int) error {
+func (f *failureLeader) Inject(clus *Cluster) error {
 	idx, err := clus.GetLeader()
 	if err != nil {
 		return err
@@ -157,7 +157,7 @@ func (f *failureLeader) Inject(clus *Cluster, round int) error {
 	return f.injectMember(clus, idx)
 }
 
-func (f *failureLeader) Recover(clus *Cluster, round int) error {
+func (f *failureLeader) Recover(clus *Cluster) error {
 	if err := f.recoverMember(clus, f.idx); err != nil {
 		return err
 	}
@@ -165,8 +165,8 @@ func (f *failureLeader) Recover(clus *Cluster, round int) error {
 	return clus.WaitHealth()
 }
 
-func (f *failureUntilSnapshot) Inject(clus *Cluster, round int) error {
-	if err := f.Failure.Inject(clus, round); err != nil {
+func (f *failureUntilSnapshot) Inject(clus *Cluster) error {
+	if err := f.Failure.Inject(clus); err != nil {
 		return err
 	}
 	if len(clus.Members) < 3 {

tools/functional-tester/tester/failure_case_no_op.go

@@ -16,8 +16,8 @@ package tester
 
 type failureNoOp failureByFunc
 
-func (f *failureNoOp) Inject(clus *Cluster, round int) error  { return nil }
-func (f *failureNoOp) Recover(clus *Cluster, round int) error { return nil }
+func (f *failureNoOp) Inject(clus *Cluster) error  { return nil }
+func (f *failureNoOp) Recover(clus *Cluster) error { return nil }
 
 func newFailureNoOp() Failure {
 	return &failureNoOp{