raft: improve behavior when snapshot fails

etcd is going to support incremental snapshot, and we design to let it
send at most one snapshot out at first stage. So when one snapshot is in
flight, snapshot request will return error.

When failing to get snapshot when sending MsgSnap, raft prints out
related log and abort sending this message.

raft/raft.go

@@ -31,6 +31,8 @@ const noLimit = math.MaxUint64
 
 var errNoLeader = errors.New("no leader")
 
+var ErrSnapshotTemporarilyUnavailable = errors.New("snapshot is temporarily unavailable")
+
 // Possible values for StateType.
 const (
 	StateFollower StateType = iota
@@ -263,6 +265,10 @@ func (r *raft) sendAppend(to uint64) {
 		m.Type = pb.MsgSnap
 		snapshot, err := r.raftLog.snapshot()
 		if err != nil {
+			if err == ErrSnapshotTemporarilyUnavailable {
+				r.logger.Debugf("%x failed to send snapshot to %x because snapshot is temporarily unavailable", r.id, to)
+				return
+			}
 			panic(err) // TODO(bdarnell)
 		}
 		if IsEmptySnap(snapshot) {

raft/storage.go

@@ -57,6 +57,9 @@ type Storage interface {
 	// first log entry is not available).
 	FirstIndex() (uint64, error)
 	// Snapshot returns the most recent snapshot.
+	// If snapshot is temporarily unavailable, it should return ErrTemporarilyUnavailable,
+	// so raft state machine could know that Storage needs some time to prepare
+	// snapshot and call Snapshot later.
 	Snapshot() (pb.Snapshot, error)
 }