Merge pull request #5596 from heyitsanthony/wal-warn-slow-fsync

wal: warn if sync exceeds a second

wal/wal.go

@@ -47,6 +47,10 @@ const (
 	// the expected size of each wal segment file.
 	// the actual size might be bigger than it.
 	segmentSizeBytes = 64 * 1000 * 1000 // 64MB
+
+	// warnSyncDuration is the amount of time allotted to an fsync before
+	// logging a warning
+	warnSyncDuration = time.Second
 )
 
 var (
@@ -393,7 +397,13 @@ func (w *WAL) sync() error {
 	}
 	start := time.Now()
 	err := fileutil.Fdatasync(w.tail().File)
-	syncDurations.Observe(time.Since(start).Seconds())
+
+	duration := time.Since(start)
+	if duration > warnSyncDuration {
+		plog.Warningf("sync duration of %v, expected less than %v", duration, warnSyncDuration)
+	}
+	syncDurations.Observe(duration.Seconds())
+
 	return err
 }