snap: add save latency metrics

snap/metrics.go

@@ -0,0 +1,30 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package snap
+
+import "github.com/coreos/etcd/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus"
+
+var (
+	// TODO: save_fsync latency?
+	// TODO: save_encoding latency?
+	saveDurations = prometheus.NewSummary(prometheus.SummaryOpts{
+		Name: "snapshot_save_total_durations_microseconds",
+		Help: "The total latency distributions of save called by snapshot.",
+	})
+)
+
+func init() {
+	prometheus.MustRegister(saveDurations)
+}

snap/snapshotter.go

@@ -24,6 +24,7 @@ import (
 	"path"
 	"sort"
 	"strings"
+	"time"
 
 	"github.com/coreos/etcd/pkg/pbutil"
 	"github.com/coreos/etcd/raft"
@@ -60,6 +61,8 @@ func (s *Snapshotter) SaveSnap(snapshot raftpb.Snapshot) error {
 }
 
 func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
+	start := time.Now()
+
 	fname := fmt.Sprintf("%016x-%016x%s", snapshot.Metadata.Term, snapshot.Metadata.Index, snapSuffix)
 	b := pbutil.MustMarshal(snapshot)
 	crc := crc32.Update(0, crcTable, b)
@@ -68,7 +71,11 @@ func (s *Snapshotter) save(snapshot *raftpb.Snapshot) error {
 	if err != nil {
 		return err
 	}
-	return ioutil.WriteFile(path.Join(s.dir, fname), d, 0666)
+	err = ioutil.WriteFile(path.Join(s.dir, fname), d, 0666)
+	if err != nil {
+		saveDurations.Observe(float64(time.Since(start).Nanoseconds() / int64(time.Microsecond)))
+	}
+	return err
 }
 
 func (s *Snapshotter) Load() (*raftpb.Snapshot, error) {