tools/functional-tester: add metrics

tools/functional-tester/etcd-tester/main.go

@@ -19,6 +19,8 @@ import (
 	"log"
 	"net/http"
 	"strings"
+
+	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus"
 )
 
 func main() {
@@ -54,6 +56,7 @@ func main() {
 
 	sh := statusHandler{status: &t.status}
 	http.Handle("/status", sh)
+	http.Handle("/metrics", prometheus.Handler())
 	go func() { log.Fatal(http.ListenAndServe(":9028", nil)) }()
 
 	t.runLoop()

tools/functional-tester/etcd-tester/metrics.go

@@ -0,0 +1,64 @@
+// Copyright 2016 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus"
+)
+
+var (
+	caseTotalCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "etcd",
+			Subsystem: "funcational_tester",
+			Name:      "case_total",
+			Help:      "Total number of finished test cases",
+		},
+		[]string{"desc"},
+	)
+
+	caseFailedTotalCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "etcd",
+			Subsystem: "funcational_tester",
+			Name:      "case_failed_total",
+			Help:      "Total number of failed test cases",
+		},
+		[]string{"desc"},
+	)
+
+	roundTotalCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: "etcd",
+			Subsystem: "funcational_tester",
+			Name:      "round_total",
+			Help:      "Total number of finished test rounds.",
+		})
+
+	roundFailedTotalCounter = prometheus.NewCounter(
+		prometheus.CounterOpts{
+			Namespace: "etcd",
+			Subsystem: "funcational_tester",
+			Name:      "round_failed_total",
+			Help:      "Total number of failed test rounds.",
+		})
+)
+
+func init() {
+	prometheus.MustRegister(caseTotalCounter)
+	prometheus.MustRegister(caseFailedTotalCounter)
+	prometheus.MustRegister(roundTotalCounter)
+	prometheus.MustRegister(roundFailedTotalCounter)
+}

tools/functional-tester/etcd-tester/tester.go

@@ -42,9 +42,12 @@ func (tt *tester) runLoop() {
 	}
 	for i := 0; i < tt.limit; i++ {
 		tt.status.setRound(i)
+		roundTotalCounter.Inc()
 
 		var currentRevision int64
 		for j, f := range tt.failures {
+			caseTotalCounter.WithLabelValues(f.Desc()).Inc()
+
 			tt.status.setCase(j)
 
 			if err := tt.cluster.WaitHealth(); err != nil {
@@ -166,6 +169,9 @@ func (tt *tester) runLoop() {
 }
 
 func (tt *tester) cleanup(i, j int) error {
+	roundFailedTotalCounter.Inc()
+	caseFailedTotalCounter.WithLabelValues(tt.failures[j].Desc()).Inc()
+
 	log.Printf("etcd-tester: [round#%d case#%d] cleaning up...", i, j)
 	if err := tt.cluster.Cleanup(); err != nil {
 		return err