etcdserver: add "etcd_server_slow_read_indexes_total"

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

etcdserver/metrics.go

@@ -71,6 +71,12 @@ var (
 		Name:      "lease_expired_total",
 		Help:      "The total number of expired leases.",
 	})
+	slowReadIndex = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "etcd",
+		Subsystem: "server",
+		Name:      "slow_read_indexes_total",
+		Help:      "The total number of pending read indexes not in sync with leader's or timed out read index requests.",
+	})
 	currentVersion = prometheus.NewGaugeVec(prometheus.GaugeOpts{
 		Namespace: "etcd",
 		Subsystem: "server",
@@ -89,6 +95,7 @@ func init() {
 	prometheus.MustRegister(proposalsPending)
 	prometheus.MustRegister(proposalsFailed)
 	prometheus.MustRegister(leaseExpired)
+	prometheus.MustRegister(slowReadIndex)
 	prometheus.MustRegister(currentVersion)
 
 	currentVersion.With(prometheus.Labels{

etcdserver/v3_server.go

@@ -655,12 +655,14 @@ func (s *EtcdServer) linearizableReadLoop() {
 						id2 = binary.BigEndian.Uint64(rs.RequestCtx)
 					}
 					plog.Warningf("ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader (request ID want %d, got %d)", id1, id2)
+					slowReadIndex.Inc()
 				}
 
 			case <-time.After(s.Cfg.ReqTimeout()):
 				plog.Warningf("timed out waiting for read index response")
 				nr.notify(ErrTimeout)
 				timeout = true
+				slowReadIndex.Inc()
 
 			case <-s.stopping:
 				return