etcd-dump-db: initial commit

tools/etcd-dump-db/README.md

@@ -0,0 +1,74 @@
+### etcd-dump-db
+
+etcd-dump-db inspects etcd db files.
+
+```
+Usage:
+  etcd-dump-db [command]
+
+Available Commands:
+  list-bucket    bucket lists all buckets.
+  iterate-bucket iterate-bucket lists key-value pairs in reverse order.
+  hash           hash computes the hash of db file.
+
+Flags:
+  -h, --help[=false]: help for etcd-dump-db
+
+Use "etcd-dump-db [command] --help" for more information about a command.
+```
+
+
+#### list-bucket [data dir or db file path]
+
+Lists all buckets.
+
+```
+$ etcd-dump-db list-bucket agent01/agent.etcd
+
+alarm
+auth
+authRoles
+authUsers
+cluster
+key
+lease
+members
+members_removed
+meta
+```
+
+
+#### hash [data dir or db file path]
+
+Computes the hash of db file.
+
+```
+$ etcd-dump-db hash agent01/agent.etcd
+db path: agent01/agent.etcd/member/snap/db
+Hash: 3700260467
+
+
+$ etcd-dump-db hash agent02/agent.etcd
+
+db path: agent02/agent.etcd/member/snap/db
+Hash: 3700260467
+
+
+$ etcd-dump-db hash agent03/agent.etcd
+
+db path: agent03/agent.etcd/member/snap/db
+Hash: 3700260467
+```
+
+
+#### iterate-bucket [data dir or db file path]
+
+Lists key-value pairs in reverse order.
+
+```
+$ etcd-dump-db iterate-bucket agent03/agent.etcd --bucket=key --limit 3
+
+key="\x00\x00\x00\x00\x005@x_\x00\x00\x00\x00\x00\x00\x00\tt", value="\n\x153640412599896088633_9"
+key="\x00\x00\x00\x00\x005@x_\x00\x00\x00\x00\x00\x00\x00\bt", value="\n\x153640412599896088633_8"
+key="\x00\x00\x00\x00\x005@x_\x00\x00\x00\x00\x00\x00\x00\at", value="\n\x153640412599896088633_7"
+```

tools/etcd-dump-db/backend.go

@@ -0,0 +1,83 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/boltdb/bolt"
+	"github.com/coreos/etcd/mvcc"
+	"github.com/coreos/etcd/mvcc/backend"
+)
+
+func snapDir(dataDir string) string {
+	return filepath.Join(dataDir, "member", "snap")
+}
+
+func getBuckets(dbPath string) (buckets []string, err error) {
+	db, derr := bolt.Open(dbPath, 0600, &bolt.Options{})
+	if derr != nil {
+		return nil, derr
+	}
+	defer db.Close()
+
+	err = db.View(func(tx *bolt.Tx) error {
+		return tx.ForEach(func(b []byte, _ *bolt.Bucket) error {
+			buckets = append(buckets, string(b))
+			return nil
+		})
+	})
+	return
+}
+
+func iterateBucket(dbPath, bucket string, limit uint64) (err error) {
+	db, derr := bolt.Open(dbPath, 0600, &bolt.Options{})
+	if derr != nil {
+		return derr
+	}
+	defer db.Close()
+
+	err = db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(bucket))
+		if b == nil {
+			return fmt.Errorf("got nil bucket for %s", bucket)
+		}
+
+		c := b.Cursor()
+
+		// iterate in reverse order (use First() and Next() for ascending order)
+		for k, v := c.Last(); k != nil; k, v = c.Prev() {
+			fmt.Printf("key=%q, value=%q\n", k, v)
+
+			limit--
+			if limit == 0 {
+				break
+			}
+		}
+
+		return nil
+	})
+	return
+}
+
+func getHash(dbPath string) (hash uint32, err error) {
+	b := backend.NewDefaultBackend(dbPath)
+	return b.Hash(mvcc.DefaultIgnores)
+}
+
+// TODO: revert by revision and find specified hash value
+// currently, it's hard because lease is in separate bucket
+// and does not modify revision

tools/etcd-dump-db/doc.go

@@ -0,0 +1,16 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// etcd-dump-db inspects etcd db files.
+package main

tools/etcd-dump-db/main.go

@@ -0,0 +1,133 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+var (
+	rootCommand = &cobra.Command{
+		Use:   "etcd-dump-db",
+		Short: "etcd-dump-db inspects etcd db files.",
+	}
+	listBucketCommand = &cobra.Command{
+		Use:   "list-bucket [data dir or db file path]",
+		Short: "bucket lists all buckets.",
+		Run:   listBucketCommandFunc,
+	}
+	iterateBucketCommand = &cobra.Command{
+		Use:   "iterate-bucket [data dir or db file path]",
+		Short: "iterate-bucket lists key-value pairs in reverse order.",
+		Run:   iterateBucketCommandFunc,
+	}
+	getHashCommand = &cobra.Command{
+		Use:   "hash [data dir or db file path]",
+		Short: "hash computes the hash of db file.",
+		Run:   getHashCommandFunc,
+	}
+)
+
+var (
+	iterateBucketName  string
+	iterateBucketLimit uint64
+
+	revertCopyPath string
+	revertRevision int64
+)
+
+func init() {
+	iterateBucketCommand.PersistentFlags().StringVar(&iterateBucketName, "bucket", "", "bucket name to iterate")
+	iterateBucketCommand.PersistentFlags().Uint64Var(&iterateBucketLimit, "limit", 0, "max number of key-value pairs to iterate (0< to iterate all)")
+
+	rootCommand.AddCommand(listBucketCommand)
+	rootCommand.AddCommand(iterateBucketCommand)
+	rootCommand.AddCommand(getHashCommand)
+}
+
+func main() {
+	if err := rootCommand.Execute(); err != nil {
+		fmt.Fprintln(os.Stdout, err)
+		os.Exit(1)
+	}
+}
+
+func listBucketCommandFunc(cmd *cobra.Command, args []string) {
+	if len(args) < 1 {
+		log.Fatalf("Must provide at least 1 argument (got %v)", args)
+	}
+	dp := args[0]
+	if !strings.HasSuffix(dp, "db") {
+		dp = filepath.Join(snapDir(dp), "db")
+	}
+	if !existFileOrDir(dp) {
+		log.Fatalf("%q does not exist", dp)
+	}
+
+	bts, err := getBuckets(dp)
+	if err != nil {
+		log.Fatal(err)
+	}
+	for _, b := range bts {
+		fmt.Println(b)
+	}
+}
+
+func iterateBucketCommandFunc(cmd *cobra.Command, args []string) {
+	if len(args) < 1 {
+		log.Fatalf("Must provide at least 1 argument (got %v)", args)
+	}
+	dp := args[0]
+	if !strings.HasSuffix(dp, "db") {
+		dp = filepath.Join(snapDir(dp), "db")
+	}
+	if !existFileOrDir(dp) {
+		log.Fatalf("%q does not exist", dp)
+	}
+
+	if iterateBucketName == "" {
+		log.Fatal("got empty bucket name")
+	}
+
+	err := iterateBucket(dp, iterateBucketName, iterateBucketLimit)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func getHashCommandFunc(cmd *cobra.Command, args []string) {
+	if len(args) < 1 {
+		log.Fatalf("Must provide at least 1 argument (got %v)", args)
+	}
+	dp := args[0]
+	if !strings.HasSuffix(dp, "db") {
+		dp = filepath.Join(snapDir(dp), "db")
+	}
+	if !existFileOrDir(dp) {
+		log.Fatalf("%q does not exist", dp)
+	}
+
+	hash, err := getHash(dp)
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("db path: %s\nHash: %d\n", dp, hash)
+}

tools/etcd-dump-db/utils.go

@@ -0,0 +1,22 @@
+// Copyright 2016 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import "os"
+
+func existFileOrDir(name string) bool {
+	_, err := os.Stat(name)
+	return err == nil
+}