Merge pull request #6072 from xiang90/tls_err

Log TLS error in health checking

cmd/Godeps/Godeps.json

@@ -197,7 +197,7 @@
 		},
 		{
 			"ImportPath": "github.com/xiang90/probing",
-			"Rev": "6a0cc1ae81b4cc11db5e491e030e4b98fba79c19"
+			"Rev": "95bd620af35406ab93d7f5bf320dba4b4667982e"
 		},
 		{
 			"ImportPath": "golang.org/x/crypto/bcrypt",

cmd/vendor/github.com/xiang90/probing/prober.go

@@ -61,7 +61,7 @@ func (p *prober) AddHTTP(id string, probingInterval time.Duration, endpoints []s
 				}
 				resp, err := p.tr.RoundTrip(req)
 				if err != nil {
-					s.recordFailure()
+					s.recordFailure(err)
 					pinned = (pinned + 1) % len(endpoints)
 					continue
 				}
@@ -71,7 +71,7 @@ func (p *prober) AddHTTP(id string, probingInterval time.Duration, endpoints []s
 				err = d.Decode(&hh)
 				resp.Body.Close()
 				if err != nil || !hh.OK {
-					s.recordFailure()
+					s.recordFailure(err)
 					pinned = (pinned + 1) % len(endpoints)
 					continue
 				}

cmd/vendor/github.com/xiang90/probing/status.go

@@ -14,6 +14,7 @@ type Status interface {
 	Total() int64
 	Loss() int64
 	Health() bool
+	Err() error
 	// Estimated smoothed round trip time
 	SRTT() time.Duration
 	// Estimated clock difference
@@ -27,6 +28,7 @@ type status struct {
 	total     int64
 	loss      int64
 	health    bool
+	err       error
 	clockdiff time.Duration
 	stopC     chan struct{}
 }
@@ -56,6 +58,12 @@ func (s *status) Health() bool {
 	return s.health
 }
 
+func (s *status) Err() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.err
+}
+
 func (s *status) ClockDiff() time.Duration {
 	s.mu.Lock()
 	defer s.mu.Unlock()
@@ -74,15 +82,17 @@ func (s *status) record(rtt time.Duration, when time.Time) {
 	s.health = true
 	s.srtt = time.Duration((1-α)*float64(s.srtt) + α*float64(rtt))
 	s.clockdiff = time.Now().Sub(when) - s.srtt/2
+	s.err = nil
 }
 
-func (s *status) recordFailure() {
+func (s *status) recordFailure(err error) {
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
 	s.total++
 	s.health = false
 	s.loss += 1
+	s.err = err
 }
 
 func (s *status) reset() {
@@ -93,4 +103,5 @@ func (s *status) reset() {
 	s.total = 0
 	s.health = false
 	s.clockdiff = 0
+	s.err = nil
 }

rafthttp/probing_status.go

@@ -25,6 +25,7 @@ var (
 	// Or the connection will time-out.
 	proberInterval           = ConnReadTimeout - time.Second
 	statusMonitoringInterval = 30 * time.Second
+	statusErrorInterval      = 5 * time.Second
 )
 
 func addPeerToProber(p probing.Prober, id string, us []string) {
@@ -44,11 +45,16 @@ func addPeerToProber(p probing.Prober, id string, us []string) {
 }
 
 func monitorProbingStatus(s probing.Status, id string) {
+	// set the first interval short to log error early.
+	interval := statusErrorInterval
 	for {
 		select {
-		case <-time.After(statusMonitoringInterval):
+		case <-time.After(interval):
 			if !s.Health() {
-				plog.Warningf("health check for peer %s could not connect", id)
+				plog.Warningf("health check for peer %s could not connect: %v", id, s.Err())
+				interval = statusErrorInterval
+			} else {
+				interval = statusMonitoringInterval
 			}
 			if s.ClockDiff() > time.Second {
 				plog.Warningf("the clock difference against peer %s is too high [%v > %v]", id, s.ClockDiff(), time.Second)