Merge pull request #8879 from brancz/adapt-rules

Adapt rules to use new gRPC metrics

Documentation/op-guide/etcd3_alert.rules

@@ -69,7 +69,7 @@ ANNOTATIONS {
 
 # alert if the 99th percentile of gRPC method calls take more than 150ms
 ALERT GRPCRequestsSlow
-IF histogram_quantile(0.99, rate(etcd_grpc_unary_requests_duration_seconds_bucket[5m])) > 0.15
+IF histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd",grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le)) > 0.15
 FOR 10m
 LABELS {
   severity = "critical"
@@ -84,8 +84,8 @@ ANNOTATIONS {
 
 # alert if more than 1% of requests to an HTTP endpoint have failed within the last 5 minutes
 ALERT HighNumberOfFailedHTTPRequests
-IF sum by(method) (rate(etcd_http_failed_total{job="etcd"}[5m]))
-  / sum by(method) (rate(etcd_http_received_total{job="etcd"}[5m])) > 0.01
+IF sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
+  / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.01
 FOR 10m
 LABELS {
   severity = "warning"
@@ -97,8 +97,8 @@ ANNOTATIONS {
 
 # alert if more than 5% of requests to an HTTP endpoint have failed within the last 5 minutes
 ALERT HighNumberOfFailedHTTPRequests
-IF sum by(method) (rate(etcd_http_failed_total{job="etcd"}[5m])) 
-  / sum by(method) (rate(etcd_http_received_total{job="etcd"}[5m])) > 0.05
+IF sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
+  / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.05
 FOR 5m
 LABELS {
   severity = "critical"

Documentation/op-guide/etcd3_alert.rules.yml

@@ -26,8 +26,8 @@ groups:
         changes within the last hour
       summary: a high number of leader changes within the etcd cluster are happening
   - alert: HighNumberOfFailedGRPCRequests
-    expr: sum(rate(etcd_grpc_requests_failed_total{job="etcd"}[5m])) BY (grpc_method)
-      / sum(rate(etcd_grpc_total{job="etcd"}[5m])) BY (grpc_method) > 0.01
+    expr: sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
+      / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.01
     for: 10m
     labels:
       severity: warning
@@ -36,8 +36,8 @@ groups:
         on etcd instance {{ $labels.instance }}'
       summary: a high number of gRPC requests are failing
   - alert: HighNumberOfFailedGRPCRequests
-    expr: sum(rate(etcd_grpc_requests_failed_total{job="etcd"}[5m])) BY (grpc_method)
-      / sum(rate(etcd_grpc_total{job="etcd"}[5m])) BY (grpc_method) > 0.05
+    expr: sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method)
+      / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.05
     for: 5m
     labels:
       severity: critical
@@ -46,7 +46,7 @@ groups:
         on etcd instance {{ $labels.instance }}'
       summary: a high number of gRPC requests are failing
   - alert: GRPCRequestsSlow
-    expr: histogram_quantile(0.99, rate(etcd_grpc_unary_requests_duration_seconds_bucket[5m]))
+    expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd",grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le))
       > 0.15
     for: 10m
     labels:

Documentation/op-guide/monitoring.md

@@ -100,7 +100,7 @@ Now Prometheus will scrape etcd metrics every 10 seconds.
 
 ### Alerting
 
-There is a set of default alerts for etcd v3 clusters for [Prometheus 1.x](./etcd3_alert.rules) as well as [Prometheus 2.x](./etcd3_alert.rules).
+There is a set of default alerts for etcd v3 clusters for [Prometheus 1.x](./etcd3_alert.rules) as well as [Prometheus 2.x](./etcd3_alert.rules.yml).
 
 > Note: `job` labels may need to be adjusted to fit a particular need. The rules were written to apply to a single cluster so it is recommended to choose labels unique to a cluster.