server: improve TestKillLeader and TestKillRandom

Remove testing on 11-node cluster due to lack of fds

etcd/etcd_functional_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package etcd
 
 import (
+	"fmt"
 	"math/rand"
 	"net/http/httptest"
 	"testing"
@@ -27,21 +28,38 @@ import (
 )
 
 func TestKillLeader(t *testing.T) {
-	tests := []int{3, 5, 9, 11}
+	tests := []int{3, 5, 9}
 
 	for i, tt := range tests {
 		es, hs := buildCluster(tt, false)
 		waitCluster(t, es)
-		waitLeader(es)
 
-		lead := es[0].p.node.Leader()
-		es[lead].Stop()
+		var totalTime time.Duration
+		for j := 0; j < tt; j++ {
+			lead, _ := waitLeader(es)
+			es[lead].Stop()
+			hs[lead].Close()
+			time.Sleep(es[0].tickDuration * defaultElection * 2)
 
-		time.Sleep(es[0].tickDuration * defaultElection * 2)
+			start := time.Now()
+			if g, _ := waitLeader(es); g == lead {
+				t.Errorf("#%d.%d: lead = %d, want not %d", i, j, g, lead)
+			}
+			take := time.Now().Sub(start)
+			totalTime += take
+			avgTime := totalTime / (time.Duration)(i+1)
+			fmt.Println("Total time:", totalTime, "; Avg time:", avgTime)
 
-		waitLeader(es)
-		if es[1].p.node.Leader() == 0 {
-			t.Errorf("#%d: lead = %d, want not 0", i, es[1].p.node.Leader())
+			c := config.New()
+			c.DataDir = es[lead].config.DataDir
+			c.Addr = hs[lead].Listener.Addr().String()
+			id := es[lead].id
+			e, h, err := buildServer(t, c, id)
+			if err != nil {
+				t.Fatal("#%d.%d: %v", i, j, err)
+			}
+			es[lead] = e
+			hs[lead] = h
 		}
 
 		destoryCluster(t, es, hs)
@@ -49,25 +67,42 @@ func TestKillLeader(t *testing.T) {
 	afterTest(t)
 }
 
-func TestRandomKill(t *testing.T) {
-	tests := []int{3, 5, 9, 11}
+func TestKillRandom(t *testing.T) {
+	tests := []int{3, 5, 9}
 
 	for _, tt := range tests {
 		es, hs := buildCluster(tt, false)
 		waitCluster(t, es)
-		waitLeader(es)
 
-		toKill := make(map[int64]struct{})
-		for len(toKill) != tt/2-1 {
-			toKill[rand.Int63n(int64(tt))] = struct{}{}
-		}
-		for k := range toKill {
-			es[k].Stop()
-		}
+		for j := 0; j < tt; j++ {
+			waitLeader(es)
+
+			toKill := make(map[int64]struct{})
+			for len(toKill) != tt/2-1 {
+				toKill[rand.Int63n(int64(tt))] = struct{}{}
+			}
+			for k := range toKill {
+				es[k].Stop()
+				hs[k].Close()
+			}
+
+			time.Sleep(es[0].tickDuration * defaultElection * 2)
 
-		time.Sleep(es[0].tickDuration * defaultElection * 2)
+			waitLeader(es)
 
-		waitLeader(es)
+			for k := range toKill {
+				c := config.New()
+				c.DataDir = es[k].config.DataDir
+				c.Addr = hs[k].Listener.Addr().String()
+				id := es[k].id
+				e, h, err := buildServer(t, c, id)
+				if err != nil {
+					t.Fatal(err)
+				}
+				es[k] = e
+				hs[k] = h
+			}
+		}
 
 		destoryCluster(t, es, hs)
 	}

etcd/etcd_test.go

@@ -20,10 +20,12 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -403,6 +405,7 @@ func initTestServer(c *config.Config, id int64, tls bool) (e *Server, h *httptes
 		}
 		c.DataDir = n
 	}
+	addr := c.Addr
 
 	srv, err := New(c)
 	if err != nil {
@@ -416,14 +419,39 @@ func initTestServer(c *config.Config, id int64, tls bool) (e *Server, h *httptes
 	m.Handle("/", e)
 	m.Handle("/raft", e.RaftHandler())
 	m.Handle("/raft/", e.RaftHandler())
-	if tls {
-		h = httptest.NewTLSServer(m)
+
+	if addr == "127.0.0.1:4001" {
+		if tls {
+			h = httptest.NewTLSServer(m)
+		} else {
+			h = httptest.NewServer(m)
+		}
 	} else {
-		h = httptest.NewServer(m)
+		var l net.Listener
+		var err error
+		for {
+			l, err = net.Listen("tcp", addr)
+			if err == nil {
+				break
+			}
+			if !strings.Contains(err.Error(), "address already in use") {
+				panic(err)
+			}
+			time.Sleep(500 * time.Millisecond)
+		}
+		h = &httptest.Server{
+			Listener: l,
+			Config:   &http.Server{Handler: m},
+		}
+		if tls {
+			h.StartTLS()
+		} else {
+			h.Start()
+		}
 	}
-
 	e.raftPubAddr = h.URL
 	e.pubAddr = h.URL
+
 	return
 }