Merge pull request #5570 from heyitsanthony/rafthttp-snapshot-tests

rafthttp: snapshot testing

pkg/ioutil/readcloser.go

@@ -14,7 +14,10 @@
 
 package ioutil
 
-import "io"
+import (
+	"fmt"
+	"io"
+)
 
 // ReaderAndCloser implements io.ReadCloser interface by combining
 // reader and closer together.
@@ -22,3 +25,42 @@ type ReaderAndCloser struct {
 	io.Reader
 	io.Closer
 }
+
+var (
+	ErrShortRead = fmt.Errorf("ioutil: short read")
+	ErrExpectEOF = fmt.Errorf("ioutil: expect EOF")
+)
+
+// NewExactReadCloser returns a ReadCloser that returns errors if the underlying
+// reader does not read back exactly the requested number of bytes.
+func NewExactReadCloser(rc io.ReadCloser, totalBytes int64) io.ReadCloser {
+	return &exactReadCloser{rc: rc, totalBytes: totalBytes}
+}
+
+type exactReadCloser struct {
+	rc         io.ReadCloser
+	br         int64
+	totalBytes int64
+}
+
+func (e *exactReadCloser) Read(p []byte) (int, error) {
+	n, err := e.rc.Read(p)
+	e.br += int64(n)
+	if e.br > e.totalBytes {
+		return 0, ErrExpectEOF
+	}
+	if e.br < e.totalBytes && n == 0 {
+		return 0, ErrShortRead
+	}
+	return n, err
+}
+
+func (e *exactReadCloser) Close() error {
+	if err := e.rc.Close(); err != nil {
+		return err
+	}
+	if e.br < e.totalBytes {
+		return ErrShortRead
+	}
+	return nil
+}

pkg/ioutil/readcloser_test.go

@@ -0,0 +1,46 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package ioutil
+
+import (
+	"bytes"
+	"io"
+	"testing"
+)
+
+type readerNilCloser struct{ io.Reader }
+
+func (rc *readerNilCloser) Close() error { return nil }
+
+// TestExactReadCloserExpectEOF expects an eof when reading too much.
+func TestExactReadCloserExpectEOF(t *testing.T) {
+	buf := bytes.NewBuffer(make([]byte, 10))
+	rc := NewExactReadCloser(&readerNilCloser{buf}, 1)
+	if _, err := rc.Read(make([]byte, 10)); err != ErrExpectEOF {
+		t.Fatalf("expected %v, got %v", ErrExpectEOF, err)
+	}
+}
+
+// TestExactReadCloserShort expects an eof when reading too little
+func TestExactReadCloserShort(t *testing.T) {
+	buf := bytes.NewBuffer(make([]byte, 5))
+	rc := NewExactReadCloser(&readerNilCloser{buf}, 10)
+	if _, err := rc.Read(make([]byte, 10)); err != nil {
+		t.Fatalf("Read expected nil err, got %v", err)
+	}
+	if err := rc.Close(); err != ErrShortRead {
+		t.Fatalf("Close expected %v, got %v", ErrShortRead, err)
+	}
+}

rafthttp/snapshot_test.go

@@ -0,0 +1,145 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package rafthttp
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/coreos/etcd/pkg/types"
+	"github.com/coreos/etcd/raft/raftpb"
+	"github.com/coreos/etcd/snap"
+)
+
+type strReaderCloser struct{ *strings.Reader }
+
+func (s strReaderCloser) Close() error { return nil }
+
+func TestSnapshotSend(t *testing.T) {
+	tests := []struct {
+		m    raftpb.Message
+		rc   io.ReadCloser
+		size int64
+
+		wsent  bool
+		wfiles int
+	}{
+		// sent and receive with no errors
+		{
+			m:    raftpb.Message{Type: raftpb.MsgSnap, To: 1},
+			rc:   strReaderCloser{strings.NewReader("hello")},
+			size: 5,
+
+			wsent:  true,
+			wfiles: 1,
+		},
+		// error when reading snapshot for send
+		{
+			m:    raftpb.Message{Type: raftpb.MsgSnap, To: 1},
+			rc:   &errReadCloser{fmt.Errorf("snapshot error")},
+			size: 1,
+
+			wsent:  false,
+			wfiles: 0,
+		},
+		// sends less than the given snapshot length
+		{
+			m:    raftpb.Message{Type: raftpb.MsgSnap, To: 1},
+			rc:   strReaderCloser{strings.NewReader("hello")},
+			size: 10000,
+
+			wsent:  false,
+			wfiles: 0,
+		},
+		// sends less than actual snapshot length
+		{
+			m:    raftpb.Message{Type: raftpb.MsgSnap, To: 1},
+			rc:   strReaderCloser{strings.NewReader("hello")},
+			size: 1,
+
+			wsent:  false,
+			wfiles: 0,
+		},
+	}
+
+	for i, tt := range tests {
+		sent, files := testSnapshotSend(t, snap.NewMessage(tt.m, tt.rc, tt.size))
+		if tt.wsent != sent {
+			t.Errorf("#%d: snapshot expected %v, got %v", i, tt.wsent, sent)
+		}
+		if tt.wfiles != len(files) {
+			t.Fatalf("#%d: expected %d files, got %d files", i, tt.wfiles, len(files))
+		}
+	}
+}
+
+func testSnapshotSend(t *testing.T, sm *snap.Message) (bool, []os.FileInfo) {
+	d, err := ioutil.TempDir(os.TempDir(), "snapdir")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(d)
+
+	r := &fakeRaft{}
+	tr := &Transport{pipelineRt: &http.Transport{}, ClusterID: types.ID(1), Raft: r}
+	ch := make(chan struct{}, 1)
+	h := &syncHandler{newSnapshotHandler(tr, r, snap.New(d), types.ID(1)), ch}
+	srv := httptest.NewServer(h)
+	defer srv.Close()
+
+	picker := mustNewURLPicker(t, []string{srv.URL})
+	snapsend := newSnapshotSender(tr, picker, types.ID(1), newPeerStatus(types.ID(1)))
+	defer snapsend.stop()
+
+	snapsend.send(*sm)
+
+	sent := false
+	select {
+	case <-time.After(time.Second):
+		t.Fatalf("timed out sending snapshot")
+	case sent = <-sm.CloseNotify():
+	}
+
+	// wait for handler to finish accepting snapshot
+	<-ch
+
+	files, rerr := ioutil.ReadDir(d)
+	if rerr != nil {
+		t.Fatal(rerr)
+	}
+	return sent, files
+}
+
+type errReadCloser struct{ err error }
+
+func (s *errReadCloser) Read(p []byte) (int, error) { return 0, s.err }
+func (s *errReadCloser) Close() error               { return s.err }
+
+type syncHandler struct {
+	h  http.Handler
+	ch chan<- struct{}
+}
+
+func (sh *syncHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	sh.h.ServeHTTP(w, r)
+	sh.ch <- struct{}{}
+}

snap/message.go

@@ -17,6 +17,7 @@ package snap
 import (
 	"io"
 
+	"github.com/coreos/etcd/pkg/ioutil"
 	"github.com/coreos/etcd/raft/raftpb"
 )
 
@@ -38,7 +39,7 @@ type Message struct {
 func NewMessage(rs raftpb.Message, rc io.ReadCloser, rcSize int64) *Message {
 	return &Message{
 		Message:    rs,
-		ReadCloser: rc,
+		ReadCloser: ioutil.NewExactReadCloser(rc, rcSize),
 		TotalSize:  int64(rs.Size()) + rcSize,
 		closeC:     make(chan bool, 1),
 	}
@@ -52,7 +53,9 @@ func (m Message) CloseNotify() <-chan bool {
 }
 
 func (m Message) CloseWithError(err error) {
-	m.ReadCloser.Close()
+	if cerr := m.ReadCloser.Close(); cerr != nil {
+		err = cerr
+	}
 	if err == nil {
 		m.closeC <- true
 	} else {