Merge pull request #1657 from xiangli-cmu/backup

*: add ctl backup support

etcdctl/command/backup_command.go

@@ -0,0 +1,87 @@
+/*
+   Copyright 2014 CoreOS, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package command
+
+import (
+	"log"
+	"math/rand"
+	"path"
+	"time"
+
+	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/codegangsta/cli"
+	"github.com/coreos/etcd/etcdserver"
+	"github.com/coreos/etcd/etcdserver/etcdserverpb"
+	"github.com/coreos/etcd/pkg/pbutil"
+	"github.com/coreos/etcd/snap"
+	"github.com/coreos/etcd/wal"
+)
+
+func NewBackupCommand() cli.Command {
+	return cli.Command{
+		Name:  "backup",
+		Usage: "backup an etcd directory",
+		Flags: []cli.Flag{
+			cli.StringFlag{Name: "data-dir", Value: "", Usage: "Path to the etcd data dir"},
+			cli.StringFlag{Name: "backup-dir", Value: "", Usage: "Path to the backup dir"},
+		},
+		Action: handleBackup,
+	}
+}
+
+// handleBackup handles a request that intends to do a backup.
+func handleBackup(c *cli.Context) {
+	srcSnap := path.Join(c.String("data-dir"), "snap")
+	destSnap := path.Join(c.String("backup-dir"), "snap")
+	srcWAL := path.Join(c.String("data-dir"), "wal")
+	destWAL := path.Join(c.String("backup-dir"), "wal")
+
+	ss := snap.New(srcSnap)
+	snapshot, err := ss.Load()
+	if err != nil && err != snap.ErrNoSnapshot {
+		log.Fatal(err)
+	}
+	var index uint64
+	if snapshot != nil {
+		index = snapshot.Index
+		newss := snap.New(destSnap)
+		newss.SaveSnap(*snapshot)
+	}
+
+	w, err := wal.OpenAtIndex(srcWAL, index)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer w.Close()
+	wmetadata, state, ents, err := w.ReadAll()
+	if err != nil {
+		log.Fatal(err)
+	}
+	var metadata etcdserverpb.Metadata
+	pbutil.MustUnmarshal(&metadata, wmetadata)
+	rand.Seed(time.Now().UnixNano())
+	metadata.NodeID = etcdserver.GenID()
+	metadata.ClusterID = etcdserver.GenID()
+
+	neww, err := wal.Create(destWAL, pbutil.MustMarshal(&metadata))
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer neww.Close()
+	if err := neww.Save(state, ents); err != nil {
+		log.Fatal(err)
+	}
+}

etcdctl/main.go

@@ -40,6 +40,7 @@ func main() {
 		cli.StringFlag{Name: "ca-file", Value: "", Usage: "verify certificates of HTTPS-enabled servers using this CA bundle"},
 	}
 	app.Commands = []cli.Command{
+		command.NewBackupCommand(),
 		command.NewMakeCommand(),
 		command.NewMakeDirCommand(),
 		command.NewRemoveCommand(),

etcdserver/force_cluster.go

@@ -17,6 +17,7 @@
 package etcdserver
 
 import (
+	"encoding/json"
 	"log"
 	"sort"
 
@@ -101,11 +102,15 @@ func getIDs(snap *raftpb.Snapshot, ents []raftpb.Entry) []uint64 {
 // createConfigChangeEnts creates a series of Raft entries (i.e.
 // EntryConfChange) to remove the set of given IDs from the cluster. The ID
 // `self` is _not_ removed, even if present in the set.
+// If `self` is not inside the given ids, it creates a Raft entry to add a
+// default member with the given `self`.
 func createConfigChangeEnts(ids []uint64, self uint64, term, index uint64) []raftpb.Entry {
 	ents := make([]raftpb.Entry, 0)
 	next := index + 1
+	found := false
 	for _, id := range ids {
 		if id == self {
+			found = true
 			continue
 		}
 		cc := &raftpb.ConfChange{
@@ -121,5 +126,27 @@ func createConfigChangeEnts(ids []uint64, self uint64, term, index uint64) []raf
 		ents = append(ents, e)
 		next++
 	}
+	if !found {
+		m := Member{
+			ID:             types.ID(self),
+			RaftAttributes: RaftAttributes{PeerURLs: []string{"http://localhost:7001", "http://localhost:2380"}},
+		}
+		ctx, err := json.Marshal(m)
+		if err != nil {
+			log.Panicf("marshal member should never fail: %v", err)
+		}
+		cc := &raftpb.ConfChange{
+			Type:    raftpb.ConfChangeAddNode,
+			NodeID:  self,
+			Context: ctx,
+		}
+		e := raftpb.Entry{
+			Type:  raftpb.EntryConfChange,
+			Data:  pbutil.MustMarshal(cc),
+			Term:  term,
+			Index: next,
+		}
+		ents = append(ents, e)
+	}
 	return ents
 }

etcdserver/force_cluster_test.go

@@ -17,10 +17,12 @@
 package etcdserver
 
 import (
+	"encoding/json"
 	"reflect"
 	"testing"
 
 	"github.com/coreos/etcd/pkg/pbutil"
+	"github.com/coreos/etcd/pkg/types"
 	"github.com/coreos/etcd/raft/raftpb"
 )
 
@@ -54,6 +56,15 @@ func TestGetIDs(t *testing.T) {
 }
 
 func TestCreateConfigChangeEnts(t *testing.T) {
+	m := Member{
+		ID:             types.ID(1),
+		RaftAttributes: RaftAttributes{PeerURLs: []string{"http://localhost:7001", "http://localhost:2380"}},
+	}
+	ctx, err := json.Marshal(m)
+	if err != nil {
+		t.Fatal(err)
+	}
+	addcc1 := &raftpb.ConfChange{Type: raftpb.ConfChangeAddNode, NodeID: 1, Context: ctx}
 	removecc2 := &raftpb.ConfChange{Type: raftpb.ConfChangeRemoveNode, NodeID: 2}
 	removecc3 := &raftpb.ConfChange{Type: raftpb.ConfChangeRemoveNode, NodeID: 3}
 	tests := []struct {
@@ -103,6 +114,17 @@ func TestCreateConfigChangeEnts(t *testing.T) {
 				{Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc3)},
 			},
 		},
+		{
+			[]uint64{2, 3},
+			1,
+			2, 2,
+
+			[]raftpb.Entry{
+				{Term: 2, Index: 3, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc2)},
+				{Term: 2, Index: 4, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(removecc3)},
+				{Term: 2, Index: 5, Type: raftpb.EntryConfChange, Data: pbutil.MustMarshal(addcc1)},
+			},
+		},
 	}
 
 	for i, tt := range tests {