Merge pull request #3609 from yichengq/raft-snapshot

raft: kill TODO about behavior when snapshot fails

raft/raft.go

@@ -31,6 +31,8 @@ const noLimit = math.MaxUint64
 
 var errNoLeader = errors.New("no leader")
 
+var ErrSnapshotTemporarilyUnavailable = errors.New("snapshot is temporarily unavailable")
+
 // Possible values for StateType.
 const (
 	StateFollower StateType = iota
@@ -260,6 +262,10 @@ func (r *raft) sendAppend(to uint64) {
 		m.Type = pb.MsgSnap
 		snapshot, err := r.raftLog.snapshot()
 		if err != nil {
+			if err == ErrSnapshotTemporarilyUnavailable {
+				r.logger.Debugf("%x failed to send snapshot to %x because snapshot is temporarily unavailable", r.id, to)
+				return
+			}
 			panic(err) // TODO(bdarnell)
 		}
 		if IsEmptySnap(snapshot) {

raft/storage.go

@@ -57,6 +57,9 @@ type Storage interface {
 	// first log entry is not available).
 	FirstIndex() (uint64, error)
 	// Snapshot returns the most recent snapshot.
+	// If snapshot is temporarily unavailable, it should return ErrTemporarilyUnavailable,
+	// so raft state machine could know that Storage needs some time to prepare
+	// snapshot and call Snapshot later.
 	Snapshot() (pb.Snapshot, error)
 }