add killRandom test

etcd_long_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"math/rand"
 	"os"
 	"strconv"
 	"strings"
@@ -34,7 +35,7 @@ func TestKillLeader(t *testing.T) {
 
 	leader := "0.0.0.0:7001"
 
-	for i := 0; i < 200; i++ {
+	for i := 0; i < 10; i++ {
 		port, _ := strconv.Atoi(strings.Split(leader, ":")[1])
 		num := port - 7001
 		fmt.Println("kill server ", num)
@@ -58,5 +59,63 @@ func TestKillLeader(t *testing.T) {
 		fmt.Println("Leader election time average is", avgTime, "with election timeout", ELECTIONTIMTOUT)
 		etcds[num], err = os.StartProcess("etcd", argGroup[num], procAttr)
 	}
+}
+
+// TestKillRandom kills random machines in the cluster and
+// restart them after all other machines agree on the same leader
+func TestKillRandom(t *testing.T) {
+	procAttr := new(os.ProcAttr)
+	procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr}
+
+	clusterSize := 9
+	argGroup, etcds, err := createCluster(clusterSize, procAttr)
+
+	if err != nil {
+		t.Fatal("cannot create cluster")
+	}
+
+	defer destroyCluster(etcds)
+
+	leaderChan := make(chan string, 1)
+
+	time.Sleep(3 * time.Second)
+
+	go leaderMonitor(clusterSize, 4, leaderChan)
+
+	toKill := make(map[int]bool)
+
+	for i := 0; i < 20; i++ {
+		fmt.Printf("TestKillRandom Round[%d/20]\n", i)
+
+		j := 0
+		for {
+
+			r := rand.Int31n(9)
+			if _, ok := toKill[int(r)]; !ok {
+				j++
+				toKill[int(r)] = true
+			}
+
+			if j > 3 {
+				break
+			}
+
+		}
+
+		for num, _ := range toKill {
+			etcds[num].Kill()
+			etcds[num].Release()
+		}
+
+		<-leaderChan
+
+		for num, _ := range toKill {
+			etcds[num], err = os.StartProcess("etcd", argGroup[num], procAttr)
+		}
+
+		toKill = make(map[int]bool)
+	}
+
+	<-leaderChan
 
 }

test.go

@@ -33,8 +33,6 @@ func set(stop chan bool) {
 				stopSet = true
 
 			default:
-				fmt.Println("Set failed!")
-				return
 			}
 		}