etcdserver: add "etcd_server_slow_read_indexes_total"

Signed-off-by: Gyuho Lee <gyuhox@gmail.com>

etcdserver/metrics.go

@@ -78,6 +78,12 @@ var (
 		Name:      "proposals_failed_total",
 		Help:      "The total number of failed proposals seen.",
 	})
+	slowReadIndex = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "etcd",
+		Subsystem: "server",
+		Name:      "slow_read_indexes_total",
+		Help:      "The total number of pending read indexes not in sync with leader's or timed out read index requests.",
+	})
 	leaseExpired = prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: "etcd_debugging",
 		Subsystem: "server",
@@ -109,6 +115,7 @@ func init() {
 	prometheus.MustRegister(proposalsApplied)
 	prometheus.MustRegister(proposalsPending)
 	prometheus.MustRegister(proposalsFailed)
+	prometheus.MustRegister(slowReadIndex)
 	prometheus.MustRegister(leaseExpired)
 	prometheus.MustRegister(quotaBackendBytes)
 	prometheus.MustRegister(currentVersion)

etcdserver/v3_server.go

@@ -689,6 +689,7 @@ func (s *EtcdServer) linearizableReadLoop() {
 					} else {
 						plog.Warningf("ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader (request ID want %d, got %d)", id1, id2)
 					}
+					slowReadIndex.Inc()
 				}
 			case <-time.After(s.Cfg.ReqTimeout()):
 				if lg != nil {
@@ -698,6 +699,7 @@ func (s *EtcdServer) linearizableReadLoop() {
 				}
 				nr.notify(ErrTimeout)
 				timeout = true
+				slowReadIndex.Inc()
 			case <-s.stopping:
 				return
 			}