etcdhttp: adjust request timeout based on config

It uses heartbeat interval and election timeout to estimate the
expected request timeout.

This PR helps etcd survive under high roundtrip-time environment,
e.g., globally-deployed cluster.

etcdmain/etcd.go

@@ -275,7 +275,7 @@ func startEtcd(cfg *config) (<-chan struct{}, error) {
 		plog.Infof("cors = %s", cfg.corsInfo)
 	}
 	ch := &cors.CORSHandler{
-		Handler: etcdhttp.NewClientHandler(s),
+		Handler: etcdhttp.NewClientHandler(s, srvcfg.ReqTimeout()),
 		Info:    cfg.corsInfo,
 	}
 	ph := etcdhttp.NewPeerHandler(s.Cluster(), s.RaftHandler())

etcdserver/config.go

@@ -111,6 +111,13 @@ func (c *ServerConfig) SnapDir() string { return path.Join(c.MemberDir(), "snap"
 
 func (c *ServerConfig) ShouldDiscover() bool { return c.DiscoveryURL != "" }
 
+// ReqTimeout returns timeout for request to finish.
+func (c *ServerConfig) ReqTimeout() time.Duration {
+	// CommitTimeout
+	// + 2 * election timeout for possible leader election
+	return c.CommitTimeout() + 2*time.Duration(c.ElectionTicks)*time.Duration(c.TickMs)*time.Millisecond
+}
+
 // CommitTimeout returns commit timeout under normal case.
 func (c *ServerConfig) CommitTimeout() time.Duration {
 	// We assume that heartbeat >= TTL.

etcdserver/etcdhttp/client.go

@@ -57,17 +57,17 @@ const (
 )
 
 // NewClientHandler generates a muxed http.Handler with the given parameters to serve etcd client requests.
-func NewClientHandler(server *etcdserver.EtcdServer) http.Handler {
+func NewClientHandler(server *etcdserver.EtcdServer, timeout time.Duration) http.Handler {
 	go capabilityLoop(server)
 
-	sec := auth.NewStore(server, defaultServerTimeout)
+	sec := auth.NewStore(server, timeout)
 
 	kh := &keysHandler{
 		sec:     sec,
 		server:  server,
 		cluster: server.Cluster(),
 		timer:   server,
-		timeout: defaultServerTimeout,
+		timeout: timeout,
 	}
 
 	sh := &statsHandler{
@@ -78,6 +78,7 @@ func NewClientHandler(server *etcdserver.EtcdServer) http.Handler {
 		sec:     sec,
 		server:  server,
 		cluster: server.Cluster(),
+		timeout: timeout,
 		clock:   clockwork.NewRealClock(),
 	}
 
@@ -176,6 +177,7 @@ type membersHandler struct {
 	sec     auth.Store
 	server  etcdserver.Server
 	cluster etcdserver.Cluster
+	timeout time.Duration
 	clock   clockwork.Clock
 }
 
@@ -189,7 +191,7 @@ func (h *membersHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 	w.Header().Set("X-Etcd-Cluster-ID", h.cluster.ID().String())
 
-	ctx, cancel := context.WithTimeout(context.Background(), defaultServerTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
 	defer cancel()
 
 	switch r.Method {

etcdserver/etcdhttp/http.go

@@ -28,12 +28,6 @@ import (
 )
 
 const (
-	// time to wait for response from EtcdServer requests
-	// 5s for disk and network delay + 10*heartbeat for commit and possible
-	// leader switch
-	// TODO: use heartbeat set in etcdserver
-	defaultServerTimeout = 5*time.Second + 10*(100*time.Millisecond)
-
 	// time to wait for a Watch request
 	defaultWatchTimeout = time.Duration(math.MaxInt64)
 )

integration/cluster_test.go

@@ -757,7 +757,7 @@ func (m *member) Launch() error {
 	for _, ln := range m.ClientListeners {
 		hs := &httptest.Server{
 			Listener: ln,
-			Config:   &http.Server{Handler: etcdhttp.NewClientHandler(m.s)},
+			Config:   &http.Server{Handler: etcdhttp.NewClientHandler(m.s, m.ServerConfig.ReqTimeout())},
 		}
 		hs.Start()
 		m.hss = append(m.hss, hs)