Browse Source

etcdserver/api/rafthttp: display roundtripper name in warnings

Signed-off-by: Gyuho Lee <leegyuho@amazon.com>
Gyuho Lee 7 years ago
parent
commit
4a239070c8
2 changed files with 14 additions and 6 deletions
  1. 12 4
      etcdserver/api/rafthttp/probing_status.go
  2. 2 2
      etcdserver/api/rafthttp/transport.go

+ 12 - 4
etcdserver/api/rafthttp/probing_status.go

@@ -17,10 +17,16 @@ package rafthttp
 import (
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/xiang90/probing"
 	"go.uber.org/zap"
 )
 
+const (
+	// RoundTripperNameSnapshot is the name of round-tripper that sends merged snapshot message.
+	RoundTripperNameSnapshot = "ROUND_TRIPPER_SNAPSHOT"
+)
+
 var (
 	// proberInterval must be shorter than read timeout.
 	// Or the connection will time-out.
@@ -29,7 +35,7 @@ var (
 	statusErrorInterval      = 5 * time.Second
 )
 
-func addPeerToProber(lg *zap.Logger, p probing.Prober, id string, us []string) {
+func addPeerToProber(lg *zap.Logger, p probing.Prober, id string, us []string, roundTripperName string, rttSecProm *prometheus.HistogramVec) {
 	hus := make([]string, len(us))
 	for i := range us {
 		hus[i] = us[i] + ProbingPrefix
@@ -47,10 +53,10 @@ func addPeerToProber(lg *zap.Logger, p probing.Prober, id string, us []string) {
 		return
 	}
 
-	go monitorProbingStatus(lg, s, id)
+	go monitorProbingStatus(lg, s, id, roundTripperName, rttSecProm)
 }
 
-func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string) {
+func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string, roundTripperName string, rttSecProm *prometheus.HistogramVec) {
 	// set the first interval short to log error early.
 	interval := statusErrorInterval
 	for {
@@ -60,6 +66,7 @@ func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string) {
 				if lg != nil {
 					lg.Warn(
 						"prober detected unhealthy status",
+						zap.String("round-tripper-name", roundTripperName),
 						zap.String("remote-peer-id", id),
 						zap.Duration("rtt", s.SRTT()),
 						zap.Error(s.Err()),
@@ -75,6 +82,7 @@ func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string) {
 				if lg != nil {
 					lg.Warn(
 						"prober found high clock drift",
+						zap.String("round-tripper-name", roundTripperName),
 						zap.String("remote-peer-id", id),
 						zap.Duration("clock-drift", s.SRTT()),
 						zap.Duration("rtt", s.ClockDiff()),
@@ -84,7 +92,7 @@ func monitorProbingStatus(lg *zap.Logger, s probing.Status, id string) {
 					plog.Warningf("the clock difference against peer %s is too high [%v > %v]", id, s.ClockDiff(), time.Second)
 				}
 			}
-			rttSec.WithLabelValues(id).Observe(s.SRTT().Seconds())
+			rttSecProm.WithLabelValues(id).Observe(s.SRTT().Seconds())
 
 		case <-s.StopNotify():
 			return

+ 2 - 2
etcdserver/api/rafthttp/transport.go

@@ -317,7 +317,7 @@ func (t *Transport) AddPeer(id types.ID, us []string) {
 	}
 	fs := t.LeaderStats.Follower(id.String())
 	t.peers[id] = startPeer(t, urls, id, fs)
-	addPeerToProber(t.Logger, t.pipelineProber, id.String(), us)
+	addPeerToProber(t.Logger, t.pipelineProber, id.String(), us, RoundTripperNameSnapshot, rttSec)
 
 	if t.Logger != nil {
 		t.Logger.Info(
@@ -389,7 +389,7 @@ func (t *Transport) UpdatePeer(id types.ID, us []string) {
 	t.peers[id].update(urls)
 
 	t.pipelineProber.Remove(id.String())
-	addPeerToProber(t.Logger, t.pipelineProber, id.String(), us)
+	addPeerToProber(t.Logger, t.pipelineProber, id.String(), us, RoundTripperNameSnapshot, rttSec)
 
 	if t.Logger != nil {
 		t.Logger.Info(