Merge branch 'fix/panic-cluster-manager' of krotik/eliasdb into master

api/v1/cluster_test.go

@@ -25,9 +25,83 @@ import (
 	"devt.de/krotik/eliasdb/api"
 	"devt.de/krotik/eliasdb/cluster"
 	"devt.de/krotik/eliasdb/cluster/manager"
+	"devt.de/krotik/eliasdb/graph"
 	"devt.de/krotik/eliasdb/graph/graphstorage"
 )
 
+func TestClusterStorage(t *testing.T) {
+	clusterQueryURL := "http://localhost" + TESTPORT + EndpointClusterQuery
+	graphURL := "http://localhost" + TESTPORT + EndpointGraph
+
+	cluster2 := createCluster(2)
+
+	joinCluster(cluster2, t)
+
+	oldGM := api.GM
+	oldGS := api.GS
+	api.GS = cluster2[0]
+	api.GM = graph.NewGraphManager(cluster2[0])
+	api.DD = cluster2[0]
+	api.DDLog = datautil.NewRingBuffer(10)
+
+	defer func() {
+		api.GM = oldGM
+		api.GS = oldGS
+		api.DD = nil
+		api.DDLog = nil
+	}()
+
+	// We should now get back a state
+
+	st, _, res := sendTestRequest(clusterQueryURL, "GET", nil)
+
+	if st != "200 OK" || res != `
+{
+  "failed": null,
+  "members": [
+    "TestClusterMember-0",
+    "localhost:9020",
+    "TestClusterMember-1",
+    "localhost:9021"
+  ],
+  "replication": 1,
+  "ts": [
+    "TestClusterMember-0",
+    "2"
+  ],
+  "tsold": [
+    "TestClusterMember-0",
+    "1"
+  ]
+}`[1:] {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Insert some data
+
+	st, _, res = sendTestRequest(graphURL+"i41health/n", "POST", []byte(`
+[{
+	"key":"3",
+	"kind":"Upload",
+	"parcel": "12345"
+}]
+`[1:]))
+
+	cluster.WaitForTransfer()
+
+	n, err := api.GM.FetchNode("i41health", "3", "Upload")
+
+	if err != nil || n.String() != `GraphNode:
+       key : 3
+      kind : Upload
+    parcel : 12345
+` {
+		t.Error("Unexpected result:", n, err)
+		return
+	}
+}
+
 func TestClusterQuery(t *testing.T) {
 	queryURL := "http://localhost" + TESTPORT + EndpointClusterQuery
 
@@ -53,9 +127,20 @@ func TestClusterQuery(t *testing.T) {
 
 	cluster2 := createCluster(2)
 
+	oldGM := api.GM
+	oldGS := api.GS
+	api.GS = cluster2[0]
+	api.GM = graph.NewGraphManager(cluster2[0])
 	api.DD = cluster2[0]
 	api.DDLog = datautil.NewRingBuffer(10)
 
+	defer func() {
+		api.GM = oldGM
+		api.GS = oldGS
+		api.DD = nil
+		api.DDLog = nil
+	}()
+
 	// We should now get back a state
 
 	st, _, res = sendTestRequest(queryURL, "GET", nil)
@@ -344,9 +429,15 @@ Create a cluster with n members (all storage is in memory)
 */
 func createCluster(n int) []*cluster.DistributedStorage {
 
+	// By default no log output
+
+	log.SetOutput(ioutil.Discard)
+
 	var mgs []*graphstorage.MemoryGraphStorage
 	var cs []*cluster.DistributedStorage
 
+	cluster.ClearMSMap()
+
 	for i := 0; i < n; i++ {
 		mgs = append(mgs, graphstorage.NewMemoryGraphStorage(fmt.Sprintf("mgs%v", i+1)).(*graphstorage.MemoryGraphStorage))
 	}
@@ -363,6 +454,26 @@ func createCluster(n int) []*cluster.DistributedStorage {
 	return cs
 }
 
+/*
+joinCluster joins up a given cluster.
+*/
+func joinCluster(cluster []*cluster.DistributedStorage, t *testing.T) {
+
+	for i, dd := range cluster {
+		dd.Start()
+		defer dd.Close()
+
+		if i > 0 {
+			err := dd.MemberManager.JoinCluster(cluster[0].MemberManager.Name(),
+				cluster[0].MemberManager.NetAddr())
+			if err != nil {
+				t.Error(err)
+				return
+			}
+		}
+	}
+}
+
 func checkStateInfo(mm *manager.MemberManager, expectedStateInfo string) error {
 	var w bytes.Buffer
 

cluster/debug.go

@@ -0,0 +1,46 @@
+package cluster
+
+import (
+	"bytes"
+	"fmt"
+	"time"
+)
+
+/*
+msmap is a map of all know  memory-only memberStorages.
+*/
+var msmap = make(map[*DistributedStorage]*memberStorage)
+
+/*
+Clear the current map of known memory-only memberStorages.
+*/
+func ClearMSMap() {
+	msmap = make(map[*DistributedStorage]*memberStorage)
+}
+
+/*
+DumpMemoryClusterLayout returns the current storage layout in a memory-only cluster
+for a given storage manager (e.g. mainPerson.nodes for Person nodes of partition main).
+*/
+func DumpMemoryClusterLayout(smname string) string {
+	buf := new(bytes.Buffer)
+
+	for _, ms := range msmap {
+		buf.WriteString(fmt.Sprintf("MemoryStorage: %s\n", ms.gs.Name()))
+		buf.WriteString(ms.dump(smname))
+	}
+
+	return buf.String()
+}
+
+/*
+WaitForTransfer waits for the datatransfer to happen.
+*/
+func WaitForTransfer() {
+	for _, ms := range msmap {
+		ms.transferWorker()
+		for ms.transferRunning {
+			time.Sleep(time.Millisecond)
+		}
+	}
+}

cluster/distributedstorage.go

@@ -100,7 +100,12 @@ that the cluster has only one member.
 func NewDistributedStorage(gs graphstorage.Storage, config map[string]interface{},
 	stateInfo manager.StateInfo) (*DistributedStorage, error) {
 
-	ds, _, err := newDistributedAndMemberStorage(gs, config, stateInfo)
+	ds, ms, err := newDistributedAndMemberStorage(gs, config, stateInfo)
+
+	if _, ok := gs.(*graphstorage.MemoryGraphStorage); ok {
+		msmap[ds] = ms // Keep track of memory storages for debugging
+	}
+
 	return ds, err
 }
 

cluster/distributedstorage_fetch_test.go

@@ -62,7 +62,7 @@ func TestSimpleDataReplicationFetch(t *testing.T) {
 
 	// Insert two strings into the store
 
-	if loc, err := sm.Insert("test1"); loc != 0 || err != nil {
+	if loc, err := sm.Insert("test1"); loc != 1 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
@@ -92,10 +92,10 @@ func TestSimpleDataReplicationFetch(t *testing.T) {
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 1666 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
@@ -103,11 +103,11 @@ cloc: 1666 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 `[1:] && res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 1666 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
-cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 1666 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
@@ -120,7 +120,7 @@ cloc: 1666 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 
 	var ret string
 
-	if err := sm.Fetch(0, &ret); err != nil || ret != "test1" {
+	if err := sm.Fetch(1, &ret); err != nil || ret != "test1" {
 		t.Error("Unexpected result:", err, ret)
 		return
 	}
@@ -139,7 +139,7 @@ cloc: 1666 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 	cluster3[0].MemberManager.StopHousekeeping = true
 	defer func() { cluster3[0].MemberManager.StopHousekeeping = false }()
 
-	if err := sm.Fetch(0, &ret); err != nil || ret != "test1" {
+	if err := sm.Fetch(1, &ret); err != nil || ret != "test1" {
 		t.Error("Unexpected result:", err, ret)
 		return
 	}
@@ -151,7 +151,7 @@ cloc: 1666 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 
 	sm = cluster3[2].StorageManager("test", false)
 
-	if err := sm.Fetch(0, &ret); err != nil || ret != "test1" {
+	if err := sm.Fetch(1, &ret); err != nil || ret != "test1" {
 		t.Error("Unexpected result:", err, ret)
 		return
 	}

cluster/distributedstorage_free_test.go

@@ -62,7 +62,7 @@ func TestSimpleDataReplicationFree(t *testing.T) {
 
 	// Insert two strings into the store
 
-	if loc, err := sm.Insert("test1"); loc != 0 || err != nil {
+	if loc, err := sm.Insert("test1"); loc != 1 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
@@ -92,10 +92,10 @@ func TestSimpleDataReplicationFree(t *testing.T) {
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 1666 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
@@ -103,11 +103,11 @@ cloc: 1666 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 `[1:] && res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 1666 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
-cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 1666 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
@@ -134,19 +134,19 @@ cloc: 1666 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 `[1:] && res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 `[1:] {
@@ -162,7 +162,7 @@ Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
 	manager.MemberErrors[cluster3[0].MemberManager.Name()] = &testNetError{}
 	cluster3[0].MemberManager.StopHousekeeping = true
 
-	sm.Free(0)
+	sm.Free(1)
 
 	// Make sure Housekeeping is running on member 1
 
@@ -171,10 +171,10 @@ Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-transfer: [TestClusterMember-0] - Free {"Loc":0,"StoreName":"test"} "null"
+transfer: [TestClusterMember-0] - Free {"Loc":1,"StoreName":"test"} "null"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 `[1:] {

cluster/distributedstorage_insert_test.go

@@ -62,7 +62,7 @@ func TestSimpleDataReplicationInsert(t *testing.T) {
 
 	// Insert two strings into the store
 
-	if loc, err := sm.Insert("test1"); loc != 0 || err != nil {
+	if loc, err := sm.Insert("test1"); loc != 1 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
@@ -92,10 +92,10 @@ func TestSimpleDataReplicationInsert(t *testing.T) {
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
@@ -103,11 +103,11 @@ cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 `[1:] && res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
-cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
@@ -118,7 +118,7 @@ cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 
 	// At this point we should have the records on the main machines and their replicas
 
-	if err := retrieveStringFromClusterLoc(ms[0], "test", 0, "test1"); err != nil {
+	if err := retrieveStringFromClusterLoc(ms[0], "test", 1, "test1"); err != nil {
 		t.Error(err)
 		return
 	}
@@ -215,10 +215,10 @@ cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 cloc: 4 (v:1) - lloc: 3 - "\b\f\x00\x05test3"
 transfer: [TestClusterMember-2] - Insert {"Loc":4,"StoreName":"test"} "\b\f\x00\x05test3"
@@ -304,10 +304,10 @@ TestClusterMember-2: [TestClusterMember-0]
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 cloc: 4 (v:1) - lloc: 3 - "\b\f\x00\x05test3"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
@@ -372,7 +372,7 @@ func TestSimpleDataReplicationInsertWithErrors(t *testing.T) {
 
 	// Insert two strings into the store
 
-	if loc, err := sm.Insert("test1"); loc != 0 || err != nil {
+	if loc, err := sm.Insert("test1"); loc != 1 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
@@ -403,55 +403,55 @@ func TestSimpleDataReplicationInsertWithErrors(t *testing.T) {
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 transfer: [TestClusterMember-3] - Insert {"Loc":2,"StoreName":"test"} "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 `[1:] && res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
-cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 transfer: [TestClusterMember-3] - Insert {"Loc":2,"StoreName":"test"} "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 `[1:] && res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 transfer: [TestClusterMember-3] - Insert {"Loc":2,"StoreName":"test"} "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
-cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 `[1:] && res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
-cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 transfer: [TestClusterMember-3] - Insert {"Loc":2,"StoreName":"test"} "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
-cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 `[1:] {
 		t.Error("Unexpected cluster storage layout: ", res)
 		return
@@ -477,14 +477,14 @@ cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-3 MemberStorageManager mgs4/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
@@ -492,14 +492,14 @@ cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 `[1:] && res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
-cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-3 MemberStorageManager mgs4/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
@@ -507,30 +507,30 @@ cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 `[1:] && res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
-cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 TestClusterMember-3 MemberStorageManager mgs4/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 `[1:] && res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
-cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
-cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
 TestClusterMember-3 MemberStorageManager mgs4/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"

cluster/distributedstorage_test.go

@@ -291,7 +291,7 @@ func TestSimpleDataDistribution(t *testing.T) {
 		return
 	}
 
-	if loc, err := sm.Insert("test1"); loc != 0 || err != nil {
+	if loc, err := sm.Insert("test1"); loc != 1 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
@@ -319,12 +319,12 @@ func TestSimpleDataDistribution(t *testing.T) {
 
 	sm.Flush()
 
-	if loc, err := sm.Insert("test4"); loc != 1 || err != nil {
+	if loc, err := sm.Insert("test4"); loc != 2 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
 
-	if loc, err := sm.Insert("test5"); loc != 2 || err != nil {
+	if loc, err := sm.Insert("test5"); loc != 4 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
@@ -333,11 +333,11 @@ func TestSimpleDataDistribution(t *testing.T) {
 
 	var res string
 
-	if err := sm.Fetch(0, &res); res != "test1" || err != nil {
+	if err := sm.Fetch(1, &res); res != "test1" || err != nil {
 		t.Error("Unexpected result:", res, err)
 		return
 	}
-	if err := sm.Fetch(1, &res); res != "test4" || err != nil {
+	if err := sm.Fetch(2, &res); res != "test4" || err != nil {
 		t.Error("Unexpected result:", res, err)
 		return
 	}
@@ -345,7 +345,7 @@ func TestSimpleDataDistribution(t *testing.T) {
 		t.Error("Unexpected result:", res, err)
 		return
 	}
-	if err := sm.Fetch(2, &res); res != "test5" || err != nil {
+	if err := sm.Fetch(4, &res); res != "test5" || err != nil {
 		t.Error("Unexpected result:", res, err)
 		return
 	}
@@ -356,7 +356,7 @@ func TestSimpleDataDistribution(t *testing.T) {
 
 	// Update some data
 
-	if err := sm.Update(0, "test11"); err != nil {
+	if err := sm.Update(1, "test11"); err != nil {
 		t.Error("Unexpected result:", err)
 		return
 	}
@@ -367,7 +367,7 @@ func TestSimpleDataDistribution(t *testing.T) {
 
 	// Lookup the data again
 
-	if err := sm.Fetch(0, &res); res != "test11" || err != nil {
+	if err := sm.Fetch(1, &res); res != "test11" || err != nil {
 		t.Error("Unexpected result:", res, err)
 		return
 	}
@@ -417,7 +417,7 @@ func TestSimpleDataDistribution(t *testing.T) {
 
 	lsm.Update(1, "test11")
 
-	if err := sm.Fetch(0, &res); err.Error() !=
+	if err := sm.Fetch(1, &res); err.Error() !=
 		"gob: decoding into local type *[]uint8, received remote type string" {
 		t.Error("Unexpected result:", res, err)
 		return
@@ -425,7 +425,7 @@ func TestSimpleDataDistribution(t *testing.T) {
 
 	// Delete some data
 
-	if err := sm.Free(0); err != nil {
+	if err := sm.Free(1); err != nil {
 		t.Error("Unexpected result:", err)
 		return
 	}
@@ -442,7 +442,7 @@ func TestSimpleDataDistribution(t *testing.T) {
 		return
 	}
 	res = ""
-	if err := sm.Fetch(2, &res); res != "test5" || err != nil {
+	if err := sm.Fetch(4, &res); res != "test5" || err != nil {
 		t.Error("Unexpected result:", res, err)
 		return
 	}

cluster/distributedstorage_update_test.go

@@ -62,7 +62,7 @@ func TestSimpleDataReplicationUpdate(t *testing.T) {
 
 	// Insert two strings into the store
 
-	if loc, err := sm.Insert("test1"); loc != 0 || err != nil {
+	if loc, err := sm.Insert("test1"); loc != 1 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
@@ -92,10 +92,10 @@ func TestSimpleDataReplicationUpdate(t *testing.T) {
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
@@ -113,7 +113,7 @@ cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 	manager.MemberErrors[cluster3[0].MemberManager.Name()] = &testNetError{}
 	cluster3[0].MemberManager.StopHousekeeping = true
 
-	if err := sm.Update(0, "test1updated"); err != nil {
+	if err := sm.Update(1, "test1updated"); err != nil {
 		t.Error("Unexpected result:", err)
 		return
 	}
@@ -124,12 +124,12 @@ cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:2) - lloc: 1 - "\x0f\f\x00\ftest1updated"
+cloc: 1 (v:2) - lloc: 1 - "\x0f\f\x00\ftest1updated"
 cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
-transfer: [TestClusterMember-0] - Update {"Loc":0,"StoreName":"test","Ver":2} "\x0f\f\x00\ftest1updated"
+transfer: [TestClusterMember-0] - Update {"Loc":1,"StoreName":"test","Ver":2} "\x0f\f\x00\ftest1updated"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
 cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
@@ -159,10 +159,10 @@ cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:2) - lloc: 1 - "\x0f\f\x00\ftest1updated"
+cloc: 1 (v:2) - lloc: 1 - "\x0f\f\x00\ftest1updated"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:2) - lloc: 1 - "\x0f\f\x00\ftest1updated"
+cloc: 1 (v:2) - lloc: 1 - "\x0f\f\x00\ftest1updated"
 cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 

cluster/distributedstoragemanager.go

@@ -43,7 +43,7 @@ Root returns a root value.
 func (dsm *DistributedStorageManager) Root(root int) uint64 {
 	var ret uint64
 
-	// Do not do anything is the cluster is not operational
+	// Do not do anything if the cluster is not operational
 
 	distTable, distTableErr := dsm.ds.DistributionTable()
 

cluster/memberaddresstable.go

@@ -168,7 +168,7 @@ func (mat *memberAddressTable) NewClusterLoc(dsname string) (uint64, error) {
 
 				ok, err = locExists(dsname, i)
 
-				if err == nil && !ok {
+				if err == nil && !ok && i != 0 {
 					ret = i
 					goto SearchResult
 
@@ -200,7 +200,7 @@ func (mat *memberAddressTable) NewClusterLoc(dsname string) (uint64, error) {
 			// Reset range counter - next time we test which if there is anything
 			// left in this range
 
-			newLocCounter = 0
+			newLocCounter = 1
 		}
 
 		mat.setNewlocCounter(dsname, newLocCounter)
@@ -334,7 +334,7 @@ func (mat *memberAddressTable) newlocCounter(dsname string) (uint64, bool, error
 
 	v, err := mat.translation.Get(newlocCounterKey(dsname))
 	if v == nil {
-		return 0, false, err
+		return 1, false, err
 	}
 
 	ret := v.(uint64)

cluster/memberaddresstable_test.go

@@ -43,7 +43,7 @@ func TestAddressTableClusterLoc(t *testing.T) {
 	cluster1, ms1 = createCluster(1, 1)
 
 	loc, err := ms1[0].at.NewClusterLoc("test1")
-	if loc != 0 || err != nil {
+	if loc != 1 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
@@ -52,7 +52,7 @@ func TestAddressTableClusterLoc(t *testing.T) {
 	// Starting an unrelated counter should have no effect
 
 	loc, err = ms1[0].at.NewClusterLoc("test2")
-	if loc != 0 || err != nil {
+	if loc != 1 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
@@ -61,32 +61,25 @@ func TestAddressTableClusterLoc(t *testing.T) {
 
 	// Advance the counter
 
-	loc, err = ms1[0].at.NewClusterLoc("test1")
-	if loc != 1 || err != nil {
-		t.Error("Unexpected result:", loc, err)
-		return
-	}
-	ms1[0].at.SetTransClusterLoc("test1", loc, 123, 1)
-
 	loc, err = ms1[0].at.NewClusterLoc("test1")
 	if loc != 2 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
-
-	// Not filling the location 2
+	ms1[0].at.SetTransClusterLoc("test1", loc, 123, 1)
 
 	loc, err = ms1[0].at.NewClusterLoc("test1")
 	if loc != 3 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
-	ms1[0].at.SetTransClusterLoc("test1", loc, 123, 1)
 
-	// The next call should find the free location 2
+	// Not filling the location 3
+
+	// The next call should find the free location 3
 
 	loc, err = ms1[0].at.NewClusterLoc("test1")
-	if loc != 2 || err != nil {
+	if loc != 3 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}

cluster/memberstorage.go

@@ -721,7 +721,7 @@ func (ms *memberStorage) dump(smname string) string {
 				k, v := it.Next()
 				key := string(k)
 
-				if strings.HasPrefix(key, transPrefix) {
+				if strings.HasPrefix(key, fmt.Sprint(transPrefix, smname, "#")) {
 					key = string(key[len(fmt.Sprint(transPrefix, smname, "#")):])
 
 					locmap[v.(*translationRec).loc] = fmt.Sprintf("%v (v:%v)",

cluster/rebalance_test.go

@@ -72,7 +72,7 @@ func TestRebalancing(t *testing.T) {
 
 	// Insert two strings into the store
 
-	if loc, err := sm.Insert("test1"); loc != 0 || err != nil {
+	if loc, err := sm.Insert("test1"); loc != 1 || err != nil {
 		t.Error("Unexpected result:", loc, err)
 		return
 	}
@@ -111,11 +111,11 @@ func TestRebalancing(t *testing.T) {
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 6 (v:1) - lloc: 2 - "\b\f\x00\x05test3"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
@@ -164,12 +164,12 @@ cloc: 6 (v:1) - lloc: 2 - "\b\f\x00\x05test3"
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 6 (v:1) - lloc: 2 - "\b\f\x00\x05test3"
 cloc: 7 (v:1) - lloc: 3 - "\b\f\x00\x05test4"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
@@ -219,12 +219,12 @@ TestClusterMember-3: [TestClusterMember-0]
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 6 (v:1) - lloc: 2 - "\b\f\x00\x05test3"
 cloc: 7 (v:1) - lloc: 3 - "\b\f\x00\x05test4"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
@@ -254,12 +254,12 @@ cloc: 6 (v:1) - lloc: 2 - "\b\f\x00\x05test3"
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 6 (v:2) - lloc: 2 - "\x10\f\x00\rtest3_updated"
 cloc: 7 (v:1) - lloc: 3 - "\b\f\x00\x05test4"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
@@ -289,12 +289,12 @@ cloc: 6 (v:1) - lloc: 2 - "\b\f\x00\x05test3"
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 6 (v:2) - lloc: 2 - "\x10\f\x00\rtest3_updated"
 cloc: 7 (v:1) - lloc: 3 - "\b\f\x00\x05test4"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
@@ -323,12 +323,12 @@ cloc: 6 (v:1) - lloc: 2 - "\b\f\x00\x05test3"
 	if res := clusterLayout(ms, "test"); res != `
 TestClusterMember-0 MemberStorageManager mgs1/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 6 (v:2) - lloc: 2 - "\x10\f\x00\rtest3_updated"
 cloc: 7 (v:1) - lloc: 3 - "\b\f\x00\x05test4"
 TestClusterMember-1 MemberStorageManager mgs2/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 
-cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
+cloc: 1 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
 cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
 TestClusterMember-2 MemberStorageManager mgs3/ls_test
 Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0 

graph/graphmanager_cluster_test.go

@@ -0,0 +1,197 @@
+/*
+ * EliasDB
+ *
+ * Copyright 2016 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+package graph
+
+import (
+	"fmt"
+	"io/ioutil"
+	"log"
+	"testing"
+
+	"devt.de/krotik/eliasdb/cluster"
+	"devt.de/krotik/eliasdb/cluster/manager"
+	"devt.de/krotik/eliasdb/graph/data"
+	"devt.de/krotik/eliasdb/graph/graphstorage"
+	"devt.de/krotik/eliasdb/hash"
+)
+
+func TestClusterStorage(t *testing.T) {
+
+	cluster2 := createCluster(2)
+
+	joinCluster(cluster2, t)
+
+	// *** Direct storage
+
+	// Insert something into a storage manager and wait for the transfer
+
+	sm := cluster2[0].StorageManager("foo", true)
+	sm2 := cluster2[1].StorageManager("foo", true)
+
+	loc, err := sm.Insert("test123")
+	if loc != 1 || err != nil {
+		t.Error("Unexpected result:", loc, err)
+		return
+	}
+
+	cluster.WaitForTransfer()
+
+	// Try to retrieve the item again
+
+	// fmt.Println(cluster.DumpMemoryClusterLayout("foo"))
+
+	var res string
+	if err := sm.Fetch(1, &res); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if res != "test123" {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	res = ""
+
+	if err := sm2.Fetch(1, &res); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if res != "test123" {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	// *** HTree storage
+
+	// Use a HTree to insert to and fetch from a storage manager
+
+	sm = cluster2[0].StorageManager("foo2", true)
+	sm2 = cluster2[1].StorageManager("foo2", true)
+
+	htree, err := hash.NewHTree(sm)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if valres, err := htree.Put([]byte("123"), "Test1"); err != nil || valres != nil {
+		t.Error("Unexpected result:", valres, err)
+		return
+	}
+
+	if valres, err := htree.Put([]byte("123"), "Test2"); err != nil || valres != "Test1" {
+		t.Error("Unexpected result:", valres, err)
+		return
+	}
+
+	// Try to retrieve the item again
+
+	cluster.WaitForTransfer()
+
+	if val, err := htree.Get([]byte("123")); err != nil || val != "Test2" {
+		t.Error("Unexpected result:", val, err)
+		return
+	}
+
+	htree2, err := hash.LoadHTree(sm2, 1)
+	if val, err := htree2.Get([]byte("123")); err != nil || val != "Test2" {
+		t.Error("Unexpected result:", val, err)
+		return
+	}
+
+	// *** GraphManager storage
+
+	gm1 := NewGraphManager(cluster2[0])
+
+	if err := gm1.StoreNode("main", data.NewGraphNodeFromMap(map[string]interface{}{
+		"key":  "123",
+		"kind": "testnode",
+		"foo":  "bar",
+	})); err != nil {
+		t.Error("Unexpected result:", err)
+		return
+	}
+
+	cluster.WaitForTransfer()
+
+	if node, err := gm1.FetchNode("main", "123", "testnode"); err != nil ||
+		node.String() != `GraphNode:
+     key : 123
+    kind : testnode
+     foo : bar
+` {
+		t.Error("Unexpected result:", node, err)
+		return
+	}
+
+	gm2 := NewGraphManager(cluster2[1])
+
+	if node, err := gm2.FetchNode("main", "123", "testnode"); err != nil ||
+		node.String() != `GraphNode:
+     key : 123
+    kind : testnode
+     foo : bar
+` {
+		t.Error("Unexpected result:", node, err)
+		return
+	}
+}
+
+/*
+Create a cluster with n members (all storage is in memory)
+*/
+func createCluster(n int) []*cluster.DistributedStorage {
+	// By default no log output
+
+	log.SetOutput(ioutil.Discard)
+
+	var mgs []*graphstorage.MemoryGraphStorage
+	var cs []*cluster.DistributedStorage
+
+	cluster.ClearMSMap()
+
+	for i := 0; i < n; i++ {
+		mgs = append(mgs, graphstorage.NewMemoryGraphStorage(fmt.Sprintf("mgs%v", i+1)).(*graphstorage.MemoryGraphStorage))
+	}
+
+	for i := 0; i < n; i++ {
+		ds, _ := cluster.NewDistributedStorage(mgs[i], map[string]interface{}{
+			manager.ConfigRPC:           fmt.Sprintf("localhost:%v", 9020+i),
+			manager.ConfigMemberName:    fmt.Sprintf("TestClusterMember-%v", i),
+			manager.ConfigClusterSecret: "test123",
+		}, manager.NewMemStateInfo())
+		cs = append(cs, ds)
+	}
+
+	return cs
+}
+
+/*
+joinCluster joins up a given cluster.
+*/
+func joinCluster(cluster []*cluster.DistributedStorage, t *testing.T) {
+
+	for i, dd := range cluster {
+		dd.Start()
+		defer dd.Close()
+
+		if i > 0 {
+			err := dd.MemberManager.JoinCluster(cluster[0].MemberManager.Name(),
+				cluster[0].MemberManager.NetAddr())
+			if err != nil {
+				t.Error(err)
+				return
+			}
+		}
+	}
+}

storage/memorystoragemanager.go

@@ -71,7 +71,7 @@ type MemoryStorageManager struct {
 	Data  map[uint64]interface{} // Map of data
 	mutex *sync.Mutex            // Mutex to protect map operations
 
-	LocCount  uint64         // Counter for locations
+	LocCount  uint64         // Counter for locations - Must start > 0
 	AccessMap map[uint64]int // Special map to simulate access issues
 }
 
@@ -79,6 +79,11 @@ type MemoryStorageManager struct {
 NewMemoryStorageManager creates a new MemoryStorageManager
 */
 func NewMemoryStorageManager(name string) *MemoryStorageManager {
+
+	// LocCount must start > 0 - so they can be stored in a Root. Roots with
+	// the value 0 are considered empty. See also graph.getHTree which will
+	// keep creating new HTrees if a Root is 0.
+
 	return &MemoryStorageManager{name, make(map[int]uint64),
 		make(map[uint64]interface{}), &sync.Mutex{}, 1, make(map[uint64]int)}
 }
@@ -91,7 +96,7 @@ func (msm *MemoryStorageManager) Name() string {
 }
 
 /*
-Root returns a root value.
+Root returns a root value. Default (empty) value is 0.
 */
 func (msm *MemoryStorageManager) Root(root int) uint64 {
 	msm.mutex.Lock()