Merge pull request #3819 from gyuho/shorten_interval_for_first_retrials

proxy: expedite proxy refresh given no endpoints

Documentation/proxy.md

@@ -8,6 +8,8 @@ The proxy will shuffle the list of cluster members periodically to avoid sending
 
 The member list used by proxy consists of all client URLs advertised within the cluster, as specified in each members' `-advertise-client-urls` flag. If this flag is set incorrectly, requests sent to the proxy are forwarded to wrong addresses and then fail. Including URLs in the `-advertise-client-urls` flag that point to the proxy itself, e.g. http://localhost:2379, is even more problematic as it will cause loops, because the proxy keeps trying to forward requests to itself until its resources (memory, file descriptors) are eventually depleted. The fix for this problem is to restart etcd member with correct `-advertise-client-urls` flag. After client URLs list in proxy is recalculated, which happens every 30 seconds, requests will be forwarded correctly.
 
+You should start the cluster machines first and then start the proxy as the proxy gets its list from reachable machines. If the proxy starts and is unable to reach a machine it will refresh aggressively. Once the machine list is non-empty the proxy will refresh at the original duration.
+
 ## Using an etcd proxy
 To start etcd in proxy mode, you need to provide three flags: `proxy`, `listen-client-urls`, and `initial-cluster` (or `discovery`). 
 

Procfile

@@ -2,5 +2,4 @@
 etcd1: bin/etcd -name infra1 -listen-client-urls http://127.0.0.1:12379 -advertise-client-urls http://127.0.0.1:12379 -listen-peer-urls http://127.0.0.1:12380 -initial-advertise-peer-urls http://127.0.0.1:12380 -initial-cluster-token etcd-cluster-1 -initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' -initial-cluster-state new
 etcd2: bin/etcd -name infra2 -listen-client-urls http://127.0.0.1:22379 -advertise-client-urls http://127.0.0.1:22379 -listen-peer-urls http://127.0.0.1:22380 -initial-advertise-peer-urls http://127.0.0.1:22380 -initial-cluster-token etcd-cluster-1 -initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' -initial-cluster-state new
 etcd3: bin/etcd -name infra3 -listen-client-urls http://127.0.0.1:32379 -advertise-client-urls http://127.0.0.1:32379 -listen-peer-urls http://127.0.0.1:32380 -initial-advertise-peer-urls http://127.0.0.1:32380 -initial-cluster-token etcd-cluster-1 -initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' -initial-cluster-state new
-# wait 3 seconds until initial cluster is ready to be proxied.
-proxy: sleep 3s && bin/etcd -name infra-proxy1 -proxy=on -listen-client-urls http://127.0.0.1:2379 -initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380'
+proxy: bin/etcd -name infra-proxy1 -proxy=on -listen-client-urls http://127.0.0.1:2379 -initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380'

proxy/director.go

@@ -22,6 +22,10 @@ import (
 	"time"
 )
 
+// defaultRefreshInterval is the default proxyRefreshIntervalMs value
+// as in etcdmain/config.go.
+const defaultRefreshInterval = 30000 * time.Millisecond
+
 func newDirector(urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterval time.Duration) *director {
 	d := &director{
 		uf:          urlsFunc,
@@ -29,9 +33,19 @@ func newDirector(urlsFunc GetProxyURLs, failureWait time.Duration, refreshInterv
 	}
 	d.refresh()
 	go func() {
+		// In order to prevent missing proxy endpoints in the first try:
+		// when given refresh interval of defaultRefreshInterval or greater
+		// and whenever there is no available proxy endpoints,
+		// give 1-second refreshInterval.
 		for {
+			ri := refreshInterval
+			if ri >= defaultRefreshInterval {
+				if len(d.endpoints()) == 0 {
+					ri = time.Second
+				}
+			}
 			select {
-			case <-time.After(refreshInterval):
+			case <-time.After(ri):
 				d.refresh()
 			}
 		}