backend: dynamically set snapshotWarningTimeout based on db size

mvcc/backend/backend.go

@@ -42,7 +42,8 @@ var (
 
 	plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "mvcc/backend")
 
-	snapshotWarningTimeout = 30 * time.Second
+	// minSnapshotWarningTimeout is the minimum threshold to trigger a long running snapshot warning.
+	minSnapshotWarningTimeout = time.Duration(30 * time.Second)
 )
 
 type Backend interface {
@@ -165,16 +166,33 @@ func (b *backend) ForceCommit() {
 }
 
 func (b *backend) Snapshot() Snapshot {
+	b.batchTx.Commit()
+
+	b.mu.RLock()
+	defer b.mu.RUnlock()
+	tx, err := b.db.Begin(false)
+	if err != nil {
+		plog.Fatalf("cannot begin tx (%s)", err)
+	}
+
 	stopc, donec := make(chan struct{}), make(chan struct{})
+	dbBytes := tx.Size()
 	go func() {
 		defer close(donec)
+		// sendRateBytes is based on transferring snapshot data over a 1 gigabit/s connection
+		// assuming a min tcp throughput of 100MB/s.
+		var sendRateBytes int64 = 100 * 1024 * 1014
+		warningTimeout := time.Duration(int64((float64(dbBytes) / float64(sendRateBytes)) * float64(time.Second)))
+		if warningTimeout < minSnapshotWarningTimeout {
+			warningTimeout = minSnapshotWarningTimeout
+		}
 		start := time.Now()
-		ticker := time.NewTicker(snapshotWarningTimeout)
+		ticker := time.NewTicker(warningTimeout)
 		defer ticker.Stop()
 		for {
 			select {
 			case <-ticker.C:
-				plog.Warningf("snapshotting is taking more than %v seconds to finish [started at %v]", time.Since(start).Seconds(), start)
+				plog.Warningf("snapshotting is taking more than %v seconds to finish transferring %v MB [started at %v]", time.Since(start).Seconds(), float64(dbBytes)/float64(1024*1014), start)
 			case <-stopc:
 				snapshotDurations.Observe(time.Since(start).Seconds())
 				return
@@ -182,14 +200,6 @@ func (b *backend) Snapshot() Snapshot {
 		}
 	}()
 
-	b.batchTx.Commit()
-
-	b.mu.RLock()
-	defer b.mu.RUnlock()
-	tx, err := b.db.Begin(false)
-	if err != nil {
-		plog.Fatalf("cannot begin tx (%s)", err)
-	}
 	return &snapshot{tx, stopc, donec}
 }