etcdserver: print out extra files in data dir instead of erroring

etcdserver/server.go

@@ -41,6 +41,7 @@ import (
 	"github.com/coreos/etcd/rafthttp"
 	"github.com/coreos/etcd/snap"
 	"github.com/coreos/etcd/store"
+	"github.com/coreos/etcd/version"
 	"github.com/coreos/etcd/wal"
 )
 
@@ -143,14 +144,16 @@ func NewServer(cfg *ServerConfig) (*EtcdServer, error) {
 	var s *raft.MemoryStorage
 	var id types.ID
 
-	walVersion, err := wal.DetectVersion(cfg.DataDir)
+	// Run the migrations.
+	dataVer, err := version.DetectDataDir(cfg.DataDir)
 	if err != nil {
 		return nil, err
 	}
-	if walVersion == wal.WALUnknown {
-		return nil, fmt.Errorf("unknown wal version in data dir %s", cfg.DataDir)
+	if err := upgradeDataDir(cfg.DataDir, cfg.Name, dataVer); err != nil {
+		return nil, err
 	}
-	haveWAL := walVersion != wal.WALNotExist
+
+	haveWAL := wal.Exist(cfg.WALDir())
 	ss := snap.New(cfg.SnapDir())
 
 	switch {
@@ -194,11 +197,6 @@ func NewServer(cfg *ServerConfig) (*EtcdServer, error) {
 		cfg.PrintWithInitial()
 		id, n, s, w = startNode(cfg, cfg.Cluster.MemberIDs())
 	case haveWAL:
-		// Run the migrations.
-		if err := upgradeWAL(cfg.DataDir, cfg.Name, walVersion); err != nil {
-			return nil, err
-		}
-
 		if err := fileutil.IsDirWriteable(cfg.DataDir); err != nil {
 			return nil, fmt.Errorf("cannot write to data directory: %v", err)
 		}

etcdserver/storage.go

@@ -25,6 +25,7 @@ import (
 	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/raft/raftpb"
 	"github.com/coreos/etcd/snap"
+	"github.com/coreos/etcd/version"
 	"github.com/coreos/etcd/wal"
 	"github.com/coreos/etcd/wal/walpb"
 )
@@ -88,9 +89,9 @@ func readWAL(waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID,
 
 // upgradeWAL converts an older version of the etcdServer data to the newest version.
 // It must ensure that, after upgrading, the most recent version is present.
-func upgradeWAL(baseDataDir string, name string, ver wal.WalVersion) error {
+func upgradeDataDir(baseDataDir string, name string, ver version.DataDirVersion) error {
 	switch ver {
-	case wal.WALv0_4:
+	case version.DataDir0_4:
 		log.Print("etcdserver: converting v0.4 log to v2.0")
 		err := migrate.Migrate4To2(baseDataDir, name)
 		if err != nil {
@@ -98,16 +99,16 @@ func upgradeWAL(baseDataDir string, name string, ver wal.WalVersion) error {
 			return err
 		}
 		fallthrough
-	case wal.WALv2_0:
+	case version.DataDir2_0:
 		err := makeMemberDir(baseDataDir)
 		if err != nil {
 			return err
 		}
 		fallthrough
-	case wal.WALv2_0_1:
+	case version.DataDir2_0_1:
 		fallthrough
 	default:
-		log.Printf("datadir is valid for the 2.0.1 format")
+		log.Printf("etcdserver: datadir is valid for the 2.0.1 format")
 	}
 	return nil
 }

test

@@ -15,7 +15,7 @@ COVER=${COVER:-"-cover"}
 source ./build
 
 # Hack: gofmt ./ will recursively check the .git directory. So use *.go for gofmt.
-TESTABLE_AND_FORMATTABLE="client discovery error etcdctl/command etcdmain etcdserver etcdserver/etcdhttp etcdserver/etcdhttp/httptypes migrate pkg/fileutil pkg/flags pkg/idutil pkg/ioutil pkg/netutil pkg/osutil pkg/pbutil pkg/types pkg/transport pkg/wait proxy raft snap store wal"
+TESTABLE_AND_FORMATTABLE="client discovery error etcdctl/command etcdmain etcdserver etcdserver/etcdhttp etcdserver/etcdhttp/httptypes migrate pkg/fileutil pkg/flags pkg/idutil pkg/ioutil pkg/netutil pkg/osutil pkg/pbutil pkg/types pkg/transport pkg/wait proxy raft snap store version wal"
 # TODO: add it to race testing when the issue is resolved
 # https://github.com/golang/go/issues/9946
 NO_RACE_TESTABLE="rafthttp"

version/version.go

@@ -14,7 +14,62 @@
 
 package version
 
+import (
+	"path"
+
+	"github.com/coreos/etcd/pkg/fileutil"
+	"github.com/coreos/etcd/pkg/types"
+)
+
 var (
 	Version         = "2.0.4+git"
 	InternalVersion = "2"
 )
+
+// WalVersion is an enum for versions of etcd logs.
+type DataDirVersion string
+
+const (
+	DataDirUnknown  DataDirVersion = "Unknown WAL"
+	DataDir0_4      DataDirVersion = "0.4.x"
+	DataDir2_0      DataDirVersion = "2.0.0"
+	DataDir2_0Proxy DataDirVersion = "2.0 proxy"
+	DataDir2_0_1    DataDirVersion = "2.0.1"
+)
+
+func DetectDataDir(dirpath string) (DataDirVersion, error) {
+	names, err := fileutil.ReadDir(dirpath)
+	if err != nil {
+		// Error reading the directory
+		return DataDirUnknown, err
+	}
+	nameSet := types.NewUnsafeSet(names...)
+	if nameSet.Contains("member") {
+		ver, err := DetectDataDir(path.Join(dirpath, "member"))
+		if ver == DataDir2_0 {
+			return DataDir2_0_1, nil
+		} else if ver == DataDir0_4 {
+			// How in the blazes did it get there?
+			return DataDirUnknown, nil
+		}
+		return ver, err
+	}
+	if nameSet.ContainsAll([]string{"snap", "wal"}) {
+		// .../wal cannot be empty to exist.
+		walnames, err := fileutil.ReadDir(path.Join(dirpath, "wal"))
+		if err == nil && len(walnames) > 0 {
+			return DataDir2_0, nil
+		}
+	}
+	if nameSet.ContainsAll([]string{"proxy"}) {
+		return DataDir2_0Proxy, nil
+	}
+	if nameSet.ContainsAll([]string{"snapshot", "conf", "log"}) {
+		return DataDir0_4, nil
+	}
+	if nameSet.ContainsAll([]string{"standby_info"}) {
+		return DataDir0_4, nil
+	}
+
+	return DataDirUnknown, nil
+}

wal/util_test.go → version/version_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package wal
+package version
 
 import (
 	"io/ioutil"
@@ -22,21 +22,20 @@ import (
 	"testing"
 )
 
-func TestDetectVersion(t *testing.T) {
+func TestDetectDataDir(t *testing.T) {
 	tests := []struct {
 		names []string
-		wver  WalVersion
+		wver  DataDirVersion
 	}{
-		{[]string{}, WALNotExist},
-		{[]string{"member/", "member/wal/", "member/wal/1", "member/snap/"}, WALv2_0_1},
-		{[]string{"snap/", "wal/", "wal/1"}, WALv2_0},
-		{[]string{"snapshot/", "conf", "log"}, WALv0_4},
-		{[]string{"weird"}, WALUnknown},
-		{[]string{"snap/", "wal/"}, WALUnknown},
+		{[]string{"member/", "member/wal/", "member/wal/1", "member/snap/"}, DataDir2_0_1},
+		{[]string{"snap/", "wal/", "wal/1"}, DataDir2_0},
+		{[]string{"snapshot/", "conf", "log"}, DataDir0_4},
+		{[]string{"weird"}, DataDirUnknown},
+		{[]string{"snap/", "wal/"}, DataDirUnknown},
 	}
 	for i, tt := range tests {
 		p := mustMakeDir(t, tt.names...)
-		ver, err := DetectVersion(p)
+		ver, err := DetectDataDir(p)
 		if ver != tt.wver {
 			t.Errorf("#%d: version = %s, want %s", i, ver, tt.wver)
 		}
@@ -45,15 +44,6 @@ func TestDetectVersion(t *testing.T) {
 		}
 		os.RemoveAll(p)
 	}
-
-	// detect on non-exist directory
-	v, err := DetectVersion(path.Join(os.TempDir(), "waltest", "not-exist"))
-	if v != WALNotExist {
-		t.Errorf("#non-exist: version = %s, want %s", v, WALNotExist)
-	}
-	if err != nil {
-		t.Errorf("#non-exist: err = %s, want %s", v, WALNotExist)
-	}
 }
 
 // mustMakeDir builds the directory that contains files with the given

wal/util.go

@@ -17,68 +17,10 @@ package wal
 import (
 	"fmt"
 	"log"
-	"os"
-	"path"
 
 	"github.com/coreos/etcd/pkg/fileutil"
-	"github.com/coreos/etcd/pkg/types"
 )
 
-// WalVersion is an enum for versions of etcd logs.
-type WalVersion string
-
-const (
-	WALUnknown   WalVersion = "Unknown WAL"
-	WALNotExist  WalVersion = "No WAL"
-	WALv0_4      WalVersion = "0.4.x"
-	WALv2_0      WalVersion = "2.0.0"
-	WALv2_0Proxy WalVersion = "2.0 proxy"
-	WALv2_0_1    WalVersion = "2.0.1"
-)
-
-func DetectVersion(dirpath string) (WalVersion, error) {
-	names, err := fileutil.ReadDir(dirpath)
-	if err != nil {
-		if os.IsNotExist(err) {
-			err = nil
-		}
-		// Error reading the directory
-		return WALNotExist, err
-	}
-	if len(names) == 0 {
-		// Empty WAL directory
-		return WALNotExist, nil
-	}
-	nameSet := types.NewUnsafeSet(names...)
-	if nameSet.Contains("member") {
-		ver, err := DetectVersion(path.Join(dirpath, "member"))
-		if ver == WALv2_0 {
-			return WALv2_0_1, nil
-		} else if ver == WALv0_4 {
-			// How in the blazes did it get there?
-			return WALUnknown, nil
-		}
-		return ver, err
-	}
-	if nameSet.ContainsAll([]string{"snap", "wal"}) {
-		// .../wal cannot be empty to exist.
-		if Exist(path.Join(dirpath, "wal")) {
-			return WALv2_0, nil
-		}
-	}
-	if nameSet.ContainsAll([]string{"proxy"}) {
-		return WALv2_0Proxy, nil
-	}
-	if nameSet.ContainsAll([]string{"snapshot", "conf", "log"}) {
-		return WALv0_4, nil
-	}
-	if nameSet.ContainsAll([]string{"standby_info"}) {
-		return WALv0_4, nil
-	}
-
-	return WALUnknown, nil
-}
-
 func Exist(dirpath string) bool {
 	names, err := fileutil.ReadDir(dirpath)
 	if err != nil {