Merge pull request #2448 from yichengq/334

tools/etcd-tester: add kill majority test

tools/functional-tester/etcd-tester/failure.go

@@ -15,11 +15,12 @@
 package main
 
 type failure interface {
-	// inject the failure into the testing cluster
-	Inject(c *cluster) error
-	// recover the injected failure and wait for the
-	// recovery of the testing cluster
-	Recover(c *cluster) error
+	// Inject injeccts the failure into the testing cluster at the given
+	// round. When calling the function, the cluster should be in health.
+	Inject(c *cluster, round int) error
+	// Recover recovers the injected failure caused by the injection of the
+	// given round and wait for the recovery of the testing cluster.
+	Recover(c *cluster, round int) error
 	// return a description of the failure
 	Desc() string
 }

tools/functional-tester/etcd-tester/failure_killall.go

@@ -24,7 +24,7 @@ func newFailureKillAll() *failureKillAll {
 	}
 }
 
-func (f *failureKillAll) Inject(c *cluster) error {
+func (f *failureKillAll) Inject(c *cluster, round int) error {
 	for _, a := range c.Agents {
 		if err := a.Stop(); err != nil {
 			return err
@@ -33,7 +33,7 @@ func (f *failureKillAll) Inject(c *cluster) error {
 	return nil
 }
 
-func (f *failureKillAll) Recover(c *cluster) error {
+func (f *failureKillAll) Recover(c *cluster, round int) error {
 	for _, a := range c.Agents {
 		if _, err := a.Restart(); err != nil {
 			return err

tools/functional-tester/etcd-tester/failure_killmaj.go

@@ -0,0 +1,57 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import "math/rand"
+
+type failureKillMajority struct {
+	description
+}
+
+func newFailureKillMajority() *failureKillMajority {
+	return &failureKillMajority{
+		description: "kill majority of the cluster",
+	}
+}
+
+func (f *failureKillMajority) Inject(c *cluster, round int) error {
+	for i := range getToKillMap(c.Size, round) {
+		if err := c.Agents[i].Stop(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (f *failureKillMajority) Recover(c *cluster, round int) error {
+	for i := range getToKillMap(c.Size, round) {
+		if _, err := c.Agents[i].Restart(); err != nil {
+			return err
+		}
+	}
+	return c.WaitHealth()
+}
+
+func getToKillMap(size int, seed int) map[int]bool {
+	m := make(map[int]bool)
+	r := rand.New(rand.NewSource(int64(seed)))
+	majority := size/2 + 1
+	for {
+		m[r.Intn(size)] = true
+		if len(m) >= majority {
+			return m
+		}
+	}
+}

tools/functional-tester/etcd-tester/failure_no.go

@@ -24,6 +24,6 @@ func newFailureBase() *failureBase {
 	}
 }
 
-func (f *failureBase) Inject(c *cluster) error { return nil }
+func (f *failureBase) Inject(c *cluster, round int) error { return nil }
 
-func (f *failureBase) Recover(c *cluster) error { return nil }
+func (f *failureBase) Recover(c *cluster, round int) error { return nil }

tools/functional-tester/etcd-tester/main.go

@@ -44,7 +44,7 @@ func main() {
 	}
 
 	t := &tester{
-		failures: []failure{newFailureBase(), newFailureKillAll()},
+		failures: []failure{newFailureBase(), newFailureKillAll(), newFailureKillMajority()},
 		cluster:  c,
 		limit:    *limit,
 	}

tools/functional-tester/etcd-tester/tester.go

@@ -35,7 +35,7 @@ func (tt *tester) runLoop() {
 			}
 			log.Printf("etcd-tester: [round#%d case#%d] start failure %s", i, j, f.Desc())
 			log.Printf("etcd-tester: [round#%d case#%d] start injecting failure...", i, j)
-			if err := f.Inject(tt.cluster); err != nil {
+			if err := f.Inject(tt.cluster, i); err != nil {
 				log.Printf("etcd-tester: [round#%d case#%d] injection error: %v", i, j, err)
 				if err := tt.cleanup(i, j); err != nil {
 					log.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)
@@ -44,7 +44,7 @@ func (tt *tester) runLoop() {
 				continue
 			}
 			log.Printf("etcd-tester: [round#%d case#%d] start recovering failure...", i, j)
-			if err := f.Recover(tt.cluster); err != nil {
+			if err := f.Recover(tt.cluster, i); err != nil {
 				log.Printf("etcd-tester: [round#%d case#%d] recovery error: %v", i, j, err)
 				if err := tt.cleanup(i, j); err != nil {
 					log.Printf("etcd-tester: [round#%d case#%d] cleanup error: %v", i, j, err)