Merge pull request #10 from evan-gu/fileSystem

added sorting, changed the interface, modified the documents, added some test case

Documentation/etcd-file-system.md

@@ -23,12 +23,13 @@ Besides the file and directory difference, all nodes have common attributes and
   The path of access control list of the node.
 
 ### Operation:
-- **Get** (path, recursive)
+- **Get** (path, recursive, sorted)
 
   Get the content of the node
     - If the node is a file, the data of the file will be returned.
     - If the node is a directory, the child nodes of the directory will be returned.
     - If recursive is true, it will recursively get the nodes of the directory.
+    - If sorted is true, the result will be sorted based on the path.
 
 - **Create** (path, value[optional], ttl [optional])
 

file_system/event.go

@@ -38,6 +38,19 @@ type KeyValuePair struct {
 	KVPairs []KeyValuePair `json:"kvs,omitempty"`
 }
 
+// interfaces for sorting
+func (k KeyValuePair) Len() int {
+	return len(k.KVPairs)
+}
+
+func (k KeyValuePair) Less(i, j int) bool {
+	return k.KVPairs[i].Key < k.KVPairs[j].Key
+}
+
+func (k KeyValuePair) Swap(i, j int) {
+	k.KVPairs[i], k.KVPairs[j] = k.KVPairs[j], k.KVPairs[i]
+}
+
 func newEvent(action string, key string, index uint64, term uint64) *Event {
 	return &Event{
 		Action: action,

file_system/file_system.go

@@ -3,6 +3,7 @@ package fileSystem
 import (
 	"fmt"
 	"path"
+	"sort"
 	"strings"
 	"time"
 
@@ -25,7 +26,7 @@ func New() *FileSystem {
 
 }
 
-func (fs *FileSystem) Get(nodePath string, recusive bool, index uint64, term uint64) (*Event, error) {
+func (fs *FileSystem) Get(nodePath string, recursive, sorted bool, index uint64, term uint64) (*Event, error) {
 	n, err := fs.InternalGet(nodePath, index, term)
 
 	if err != nil {
@@ -50,13 +51,22 @@ func (fs *FileSystem) Get(nodePath string, recusive bool, index uint64, term uin
 				continue
 			}
 
-			e.KVPairs[i] = child.Pair(recusive)
+			e.KVPairs[i] = child.Pair(recursive, sorted)
 
 			i++
 		}
 
 		// eliminate hidden nodes
 		e.KVPairs = e.KVPairs[:i]
+
+		rootPairs := KeyValuePair{
+			KVPairs: e.KVPairs,
+		}
+
+		if sorted {
+			sort.Sort(rootPairs)
+		}
+
 	} else { // node is file
 		e.Value = n.Value
 	}

file_system/file_system_test.go

@@ -1,6 +1,8 @@
 package fileSystem
 
 import (
+	"math/rand"
+	"strconv"
 	"testing"
 	"time"
 )
@@ -34,7 +36,7 @@ func TestCreateAndGet(t *testing.T) {
 		t.Fatal("Cannot create /fooDir")
 	}
 
-	e, err := fs.Get("/fooDir", false, 3, 1)
+	e, err := fs.Get("/fooDir", false, false, 3, 1)
 
 	if err != nil || e.Dir != true {
 		t.Fatal("Cannot create /fooDir ")
@@ -64,7 +66,7 @@ func TestUpdateFile(t *testing.T) {
 		t.Fatalf("cannot update %s=barbar [%s]", "/foo/bar", err.Error())
 	}
 
-	e, err := fs.Get("/foo/bar", false, 2, 1)
+	e, err := fs.Get("/foo/bar", false, false, 2, 1)
 
 	if err != nil {
 		t.Fatalf("cannot get %s [%s]", "/foo/bar", err.Error())
@@ -106,7 +108,7 @@ func TestUpdateFile(t *testing.T) {
 
 	// sleep 50ms, it should still reach the node
 	time.Sleep(time.Microsecond * 50)
-	e, err = fs.Get("/foo/foo", true, 7, 1)
+	e, err = fs.Get("/foo/foo", true, false, 7, 1)
 
 	if err != nil || e.Key != "/foo/foo" {
 		t.Fatalf("cannot get dir before expiration [%s]", err.Error())
@@ -126,23 +128,23 @@ func TestUpdateFile(t *testing.T) {
 
 	// wait for expiration
 	time.Sleep(time.Second * 3)
-	e, err = fs.Get("/foo/foo", true, 7, 1)
+	e, err = fs.Get("/foo/foo", true, false, 7, 1)
 
 	if err == nil {
 		t.Fatal("still can get dir after expiration [%s]")
 	}
 
-	_, err = fs.Get("/foo/foo/foo1", true, 7, 1)
+	_, err = fs.Get("/foo/foo/foo1", true, false, 7, 1)
 	if err == nil {
 		t.Fatal("still can get sub node after expiration [%s]")
 	}
 
-	_, err = fs.Get("/foo/foo/foo2", true, 7, 1)
+	_, err = fs.Get("/foo/foo/foo2", true, false, 7, 1)
 	if err == nil {
 		t.Fatal("still can get sub dir after expiration [%s]")
 	}
 
-	_, err = fs.Get("/foo/foo/foo2/boo", true, 7, 1)
+	_, err = fs.Get("/foo/foo/foo2/boo", true, false, 7, 1)
 	if err == nil {
 		t.Fatalf("still can get sub node of sub dir after expiration [%s]", err.Error())
 	}
@@ -160,7 +162,7 @@ func TestListDirectory(t *testing.T) {
 	// set key-value /foo/fooDir/foo=bar
 	fs.Create("/foo/fooDir/foo", "bar", Permanent, 2, 1)
 
-	e, err := fs.Get("/foo", true, 2, 1)
+	e, err := fs.Get("/foo", true, false, 2, 1)
 
 	if err != nil {
 		t.Fatalf("%v", err)
@@ -187,7 +189,7 @@ func TestListDirectory(t *testing.T) {
 	// set key-value /foo/_hidden/foo -> bar
 	fs.Create("/foo/_hidden/foo", "bar", Permanent, 3, 1)
 
-	e, _ = fs.Get("/foo", false, 2, 1)
+	e, _ = fs.Get("/foo", false, false, 2, 1)
 
 	if len(e.KVPairs) != 2 {
 		t.Fatalf("hidden node is not hidden! %s", e.KVPairs[2].Key)
@@ -204,7 +206,7 @@ func TestRemove(t *testing.T) {
 		t.Fatalf("cannot delete %s [%s]", "/foo", err.Error())
 	}
 
-	_, err = fs.Get("/foo", false, 1, 1)
+	_, err = fs.Get("/foo", false, false, 1, 1)
 
 	if err == nil || err.Error() != "Key Not Found" {
 		t.Fatalf("can get the node after deletion")
@@ -226,7 +228,7 @@ func TestRemove(t *testing.T) {
 		t.Fatalf("cannot delete %s [%s]", "/foo", err.Error())
 	}
 
-	_, err = fs.Get("/foo", false, 1, 1)
+	_, err = fs.Get("/foo", false, false, 1, 1)
 
 	if err == nil || err.Error() != "Key Not Found" {
 		t.Fatalf("can get the node after deletion ")
@@ -382,7 +384,7 @@ func createAndGet(fs *FileSystem, path string, t *testing.T) {
 		t.Fatalf("cannot create %s=bar [%s]", path, err.Error())
 	}
 
-	e, err := fs.Get(path, false, 1, 1)
+	e, err := fs.Get(path, false, false, 1, 1)
 
 	if err != nil {
 		t.Fatalf("cannot get %s [%s]", path, err.Error())
@@ -402,3 +404,77 @@ func nonblockingRetrive(c <-chan *Event) *Event {
 		return nil
 	}
 }
+
+func TestSort(t *testing.T) {
+	fs := New()
+
+	// simulating random creation
+	keys := GenKeys(80, 4)
+
+	//t.Log(keys)
+	i := uint64(1)
+	for _, k := range keys {
+		_, err := fs.Create(k, "bar", Permanent, i, 1)
+		if err != nil {
+			//t.Logf("create node[%s] failed %s", k, err.Error())
+		} else {
+			i++
+		}
+	}
+
+	e, err := fs.Get("/foo", true, true, i, 1)
+	if err != nil {
+		t.Fatalf("get dir nodes failed [%s]", err.Error())
+	}
+
+	for i, k := range e.KVPairs[:len(e.KVPairs)-1] {
+		//t.Log("root:")
+		//t.Log(k)
+		if k.Key >= e.KVPairs[i+1].Key {
+			t.Fatalf("sort failed, [%s] should be placed after [%s]", k.Key, e.KVPairs[i+1].Key)
+		}
+
+		if k.Dir {
+			recursiveTestSort(k, t)
+		}
+
+	}
+
+	if k := e.KVPairs[len(e.KVPairs)-1]; k.Dir {
+		recursiveTestSort(k, t)
+	}
+}
+
+func recursiveTestSort(k KeyValuePair, t *testing.T) {
+	//t.Log("recursive in")
+	//t.Log(k)
+	for i, v := range k.KVPairs[:len(k.KVPairs)-1] {
+		if v.Key >= k.KVPairs[i+1].Key {
+			t.Fatalf("sort failed, [%s] should be placed after [%s]", v.Key, k.KVPairs[i+1].Key)
+		}
+
+		if v.Dir {
+			recursiveTestSort(v, t)
+		}
+
+	}
+
+	if v := k.KVPairs[len(k.KVPairs)-1]; v.Dir {
+		recursiveTestSort(v, t)
+	}
+}
+
+// GenKeys randomly generate num of keys with max depth
+func GenKeys(num int, depth int) []string {
+	keys := make([]string, num)
+	for i := 0; i < num; i++ {
+
+		keys[i] = "/foo"
+		depth := rand.Intn(depth) + 1
+
+		for j := 0; j < depth; j++ {
+			keys[i] += "/" + strconv.Itoa(rand.Int()%20)
+		}
+	}
+	return keys
+}

file_system/node.go

@@ -3,6 +3,7 @@ package fileSystem
 import (
 	"fmt"
 	"path"
+	"sort"
 	"sync"
 	"time"
 
@@ -285,7 +286,7 @@ func (n *Node) IsHidden() bool {
 	return false
 }
 
-func (n *Node) Pair(recurisive bool) KeyValuePair {
+func (n *Node) Pair(recurisive, sorted bool) KeyValuePair {
 
 	if n.IsDir() {
 		pair := KeyValuePair{
@@ -309,14 +310,16 @@ func (n *Node) Pair(recurisive bool) KeyValuePair {
 				continue
 			}
 
-			pair.KVPairs[i] = child.Pair(recurisive)
+			pair.KVPairs[i] = child.Pair(recurisive, sorted)
 
 			i++
 		}
 
 		// eliminate hidden nodes
 		pair.KVPairs = pair.KVPairs[:i]
-
+		if sorted {
+			sort.Sort(pair)
+		}
 		return pair
 	}
 

store/test.go

@@ -10,11 +10,11 @@ func GenKeys(num int, depth int) []string {
 	keys := make([]string, num)
 	for i := 0; i < num; i++ {
 
-		keys[i] = "/foo/"
+		keys[i] = "/foo"
 		depth := rand.Intn(depth) + 1
 
 		for j := 0; j < depth; j++ {
-			keys[i] += "/" + strconv.Itoa(rand.Int())
+			keys[i] += "/" + strconv.Itoa(rand.Int()%20)
 		}
 	}
 	return keys