distributedstorage_insert_test.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541
  1. /*
  2. * EliasDB
  3. *
  4. * Copyright 2016 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the Mozilla Public
  7. * License, v. 2.0. If a copy of the MPL was not distributed with this
  8. * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  9. */
  10. package cluster
  11. import (
  12. "math"
  13. "testing"
  14. "time"
  15. "devt.de/krotik/eliasdb/cluster/manager"
  16. )
  17. func TestSimpleDataReplicationInsert(t *testing.T) {
  18. // Set a low distribution range
  19. defaultDistributionRange = 10
  20. defer func() { defaultDistributionRange = math.MaxUint64 }()
  21. // Setup a cluster
  22. manager.FreqHousekeeping = 5
  23. defer func() { manager.FreqHousekeeping = 1000 }()
  24. // Log transfer worker runs
  25. logTransferWorker = true
  26. defer func() { logTransferWorker = false }()
  27. // Create a cluster with 3 members and a replication factor of 2
  28. cluster3, ms := createCluster(3, 2)
  29. // Debug output
  30. // manager.LogDebug = manager.LogInfo
  31. // log.SetOutput(os.Stderr)
  32. // defer func() { log.SetOutput(ioutil.Discard) }()
  33. for i, dd := range cluster3 {
  34. dd.Start()
  35. defer dd.Close()
  36. if i > 0 {
  37. err := dd.MemberManager.JoinCluster(cluster3[0].MemberManager.Name(), cluster3[0].MemberManager.NetAddr())
  38. if err != nil {
  39. t.Error(err)
  40. return
  41. }
  42. }
  43. }
  44. sm := cluster3[1].StorageManager("test", true)
  45. // Insert two strings into the store
  46. if loc, err := sm.Insert("test1"); loc != 0 || err != nil {
  47. t.Error("Unexpected result:", loc, err)
  48. return
  49. }
  50. sm.Flush()
  51. time.Sleep(10 * time.Millisecond)
  52. if loc, err := sm.Insert("test2"); loc != 3 || err != nil {
  53. t.Error("Unexpected result:", loc, err)
  54. return
  55. }
  56. sm.Flush()
  57. // Ensure the transfer worker is running on all members
  58. for _, m := range ms {
  59. m.transferWorker()
  60. for m.transferRunning {
  61. time.Sleep(time.Millisecond)
  62. }
  63. }
  64. // Check that we have a certain storage layout in the cluster
  65. if res := clusterLayout(ms, "test"); res != `
  66. TestClusterMember-0 MemberStorageManager mgs1/ls_test
  67. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  68. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  69. TestClusterMember-1 MemberStorageManager mgs2/ls_test
  70. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  71. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  72. cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
  73. TestClusterMember-2 MemberStorageManager mgs3/ls_test
  74. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  75. cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  76. `[1:] && res != `
  77. TestClusterMember-0 MemberStorageManager mgs1/ls_test
  78. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  79. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  80. TestClusterMember-1 MemberStorageManager mgs2/ls_test
  81. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  82. cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  83. cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
  84. TestClusterMember-2 MemberStorageManager mgs3/ls_test
  85. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  86. cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  87. `[1:] {
  88. t.Error("Unexpected cluster storage layout: ", res)
  89. return
  90. }
  91. // At this point we should have the records on the main machines and their replicas
  92. if err := retrieveStringFromClusterLoc(ms[0], "test", 0, "test1"); err != nil {
  93. t.Error(err)
  94. return
  95. }
  96. // Simulate a failure on members 0 and 2
  97. manager.MemberErrors = make(map[string]error)
  98. defer func() { manager.MemberErrors = nil }()
  99. if err := checkStateInfo(cluster3[1].MemberManager, `
  100. {
  101. "failed": null,
  102. "members": [
  103. "TestClusterMember-1",
  104. "localhost:9021",
  105. "TestClusterMember-0",
  106. "localhost:9020",
  107. "TestClusterMember-2",
  108. "localhost:9022"
  109. ],
  110. "replication": 2
  111. }
  112. `[1:]); err != nil {
  113. t.Error(err)
  114. return
  115. }
  116. if !cluster3[1].IsOperational() {
  117. t.Error("Cluster should be operational at this point")
  118. return
  119. }
  120. if res := cluster3[1].ReplicationFactor(); res != 2 {
  121. t.Error("Unexpected result:", res)
  122. return
  123. }
  124. // Simulate a failure on members 0 and 2
  125. manager.MemberErrors[cluster3[0].MemberManager.Name()] = &testNetError{}
  126. cluster3[0].MemberManager.StopHousekeeping = true
  127. defer func() { cluster3[0].MemberManager.StopHousekeeping = false }()
  128. manager.MemberErrors[cluster3[2].MemberManager.Name()] = &testNetError{}
  129. cluster3[2].MemberManager.StopHousekeeping = true
  130. // Since members 0 and 2 are not reachable the system should choose member 1
  131. if loc, err := sm.Insert("test3"); loc != 4 || err != nil {
  132. t.Error("Unexpected result:", loc, err)
  133. return
  134. }
  135. // Make sure Housekeeping is running
  136. cluster3[1].MemberManager.HousekeepingWorker()
  137. time.Sleep(10 * time.Microsecond)
  138. // Check that the cluster has recorded the failure
  139. if err := checkStateInfo(cluster3[1].MemberManager, `
  140. {
  141. "failed": [
  142. "TestClusterMember-0",
  143. "test.net.Error",
  144. "TestClusterMember-2",
  145. "test.net.Error"
  146. ],
  147. "members": [
  148. "TestClusterMember-1",
  149. "localhost:9021",
  150. "TestClusterMember-0",
  151. "localhost:9020",
  152. "TestClusterMember-2",
  153. "localhost:9022"
  154. ],
  155. "replication": 2
  156. }
  157. `[1:]); err != nil {
  158. t.Error(err)
  159. return
  160. }
  161. // Ensure the transfer worker is running on member 1
  162. ms[1].transferWorker()
  163. // Check that we have a certain storage layout in the cluster
  164. // The test3 record is stored on member 1. Member 1 should also have
  165. // a pending transfer request for member 2 which is for the
  166. // replication of the test3 record
  167. if res := clusterLayout(ms, "test"); res != `
  168. TestClusterMember-0 MemberStorageManager mgs1/ls_test
  169. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  170. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  171. TestClusterMember-1 MemberStorageManager mgs2/ls_test
  172. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  173. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  174. cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
  175. cloc: 4 (v:1) - lloc: 3 - "\b\f\x00\x05test3"
  176. transfer: [TestClusterMember-2] - Insert {"Loc":4,"StoreName":"test"} "\b\f\x00\x05test3"
  177. TestClusterMember-2 MemberStorageManager mgs3/ls_test
  178. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  179. cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  180. `[1:] {
  181. t.Error("Unexpected cluster storage layout: ", res)
  182. return
  183. }
  184. // Although the insert succeeded the cluster should now be reported as broken
  185. if ms[1].ds.distributionTable != nil {
  186. t.Error("Distribution table should not exist in a broken cluster")
  187. return
  188. }
  189. if loc, err := sm.Insert("test4"); err.Error() != "Storage disabled: Too many members failed (total: 3, failed: 2, replication: 2)" {
  190. t.Error("Unexpected result:", loc, err)
  191. return
  192. }
  193. if cluster3[1].IsOperational() {
  194. t.Error("Cluster should not be operational at this point")
  195. return
  196. }
  197. if res := cluster3[1].ReplicationFactor(); res != 0 {
  198. t.Error("Unexpected result:", res)
  199. return
  200. }
  201. // Now make member 2 work again
  202. delete(manager.MemberErrors, cluster3[2].MemberManager.Name())
  203. cluster3[2].MemberManager.StopHousekeeping = false
  204. // Make sure Housekeeping was running on all available members
  205. cluster3[1].MemberManager.HousekeepingWorker()
  206. cluster3[2].MemberManager.HousekeepingWorker()
  207. // Check that the cluster has recovered
  208. if err := checkStateInfo(cluster3[2].MemberManager, `
  209. {
  210. "failed": [
  211. "TestClusterMember-0",
  212. "test.net.Error"
  213. ],
  214. "members": [
  215. "TestClusterMember-2",
  216. "localhost:9022",
  217. "TestClusterMember-0",
  218. "localhost:9020",
  219. "TestClusterMember-1",
  220. "localhost:9021"
  221. ],
  222. "replication": 2
  223. }
  224. `[1:]); err != nil {
  225. t.Error(err)
  226. return
  227. }
  228. if dss := ms[1].ds.distributionTable.String(); dss != `
  229. Location ranges:
  230. TestClusterMember-0: 0 -> 2
  231. TestClusterMember-1: 3 -> 5
  232. TestClusterMember-2: 6 -> 10
  233. Replicas (factor=2) :
  234. TestClusterMember-0: [TestClusterMember-1]
  235. TestClusterMember-1: [TestClusterMember-2]
  236. TestClusterMember-2: [TestClusterMember-0]
  237. `[1:] {
  238. t.Error("Unexpected distribution table:", dss)
  239. return
  240. }
  241. // Check that replication has happened
  242. if res := clusterLayout(ms, "test"); res != `
  243. TestClusterMember-0 MemberStorageManager mgs1/ls_test
  244. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  245. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  246. TestClusterMember-1 MemberStorageManager mgs2/ls_test
  247. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  248. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  249. cloc: 3 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
  250. cloc: 4 (v:1) - lloc: 3 - "\b\f\x00\x05test3"
  251. TestClusterMember-2 MemberStorageManager mgs3/ls_test
  252. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  253. cloc: 3 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  254. cloc: 4 (v:1) - lloc: 2 - "\b\f\x00\x05test3"
  255. `[1:] {
  256. t.Error("Unexpected cluster storage layout: ", res)
  257. return
  258. }
  259. }
  260. func TestSimpleDataReplicationInsertWithErrors(t *testing.T) {
  261. // Set a low distribution range
  262. defaultDistributionRange = 10
  263. defer func() { defaultDistributionRange = math.MaxUint64 }()
  264. // Setup a cluster
  265. manager.FreqHousekeeping = 5
  266. defer func() { manager.FreqHousekeeping = 1000 }()
  267. // Log transfer worker runs
  268. logTransferWorker = true
  269. defer func() { logTransferWorker = false }()
  270. // Create a cluster with 4 members and a replication factor of 3
  271. cluster4, ms := createCluster(4, 3)
  272. // Debug output
  273. // manager.LogDebug = manager.LogInfo
  274. // log.SetOutput(os.Stderr)
  275. // defer func() { log.SetOutput(ioutil.Discard) }()
  276. for i, dd := range cluster4 {
  277. dd.Start()
  278. defer dd.Close()
  279. if i > 0 {
  280. err := dd.MemberManager.JoinCluster(cluster4[0].MemberManager.Name(), cluster4[0].MemberManager.NetAddr())
  281. if err != nil {
  282. t.Error(err)
  283. return
  284. }
  285. }
  286. }
  287. // Simulate member 3 failing
  288. manager.MemberErrors = make(map[string]error)
  289. defer func() { manager.MemberErrors = nil }()
  290. manager.MemberErrors[cluster4[3].MemberManager.Name()] = &testNetError{}
  291. cluster4[3].MemberManager.StopHousekeeping = true
  292. sm := cluster4[1].StorageManager("test", true)
  293. // Insert two strings into the store
  294. if loc, err := sm.Insert("test1"); loc != 0 || err != nil {
  295. t.Error("Unexpected result:", loc, err)
  296. return
  297. }
  298. sm.Flush()
  299. time.Sleep(10 * time.Millisecond)
  300. if loc, err := sm.Insert("test2"); loc != 2 || err != nil {
  301. t.Error("Unexpected result:", loc, err)
  302. return
  303. }
  304. sm.Flush()
  305. // Ensure the transfer worker is running on all members
  306. for _, m := range ms {
  307. m.transferWorker()
  308. for m.transferRunning {
  309. time.Sleep(time.Millisecond)
  310. }
  311. }
  312. // Check that we have a certain storage layout in the cluster
  313. // The transfer request has partially succeeded
  314. if res := clusterLayout(ms, "test"); res != `
  315. TestClusterMember-0 MemberStorageManager mgs1/ls_test
  316. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  317. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  318. TestClusterMember-1 MemberStorageManager mgs2/ls_test
  319. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  320. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  321. cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
  322. transfer: [TestClusterMember-3] - Insert {"Loc":2,"StoreName":"test"} "\b\f\x00\x05test2"
  323. TestClusterMember-2 MemberStorageManager mgs3/ls_test
  324. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  325. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  326. cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
  327. `[1:] && res != `
  328. TestClusterMember-0 MemberStorageManager mgs1/ls_test
  329. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  330. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  331. TestClusterMember-1 MemberStorageManager mgs2/ls_test
  332. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  333. cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  334. cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
  335. transfer: [TestClusterMember-3] - Insert {"Loc":2,"StoreName":"test"} "\b\f\x00\x05test2"
  336. TestClusterMember-2 MemberStorageManager mgs3/ls_test
  337. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  338. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  339. cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
  340. `[1:] && res != `
  341. TestClusterMember-0 MemberStorageManager mgs1/ls_test
  342. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  343. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  344. TestClusterMember-1 MemberStorageManager mgs2/ls_test
  345. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  346. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  347. cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
  348. transfer: [TestClusterMember-3] - Insert {"Loc":2,"StoreName":"test"} "\b\f\x00\x05test2"
  349. TestClusterMember-2 MemberStorageManager mgs3/ls_test
  350. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  351. cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  352. cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
  353. `[1:] && res != `
  354. TestClusterMember-0 MemberStorageManager mgs1/ls_test
  355. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  356. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  357. TestClusterMember-1 MemberStorageManager mgs2/ls_test
  358. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  359. cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  360. cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
  361. transfer: [TestClusterMember-3] - Insert {"Loc":2,"StoreName":"test"} "\b\f\x00\x05test2"
  362. TestClusterMember-2 MemberStorageManager mgs3/ls_test
  363. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  364. cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  365. cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
  366. `[1:] {
  367. t.Error("Unexpected cluster storage layout: ", res)
  368. return
  369. }
  370. // Simulate member 3 working again
  371. delete(manager.MemberErrors, cluster4[3].MemberManager.Name())
  372. cluster4[3].MemberManager.StopHousekeeping = false
  373. // Ensure the transfer worker is running on all members
  374. for _, m := range ms {
  375. m.transferWorker()
  376. for m.transferRunning {
  377. time.Sleep(time.Millisecond)
  378. }
  379. }
  380. // Check that we have a certain storage layout in the cluster
  381. // The transfer request has now fully succeeded
  382. if res := clusterLayout(ms, "test"); res != `
  383. TestClusterMember-0 MemberStorageManager mgs1/ls_test
  384. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  385. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  386. TestClusterMember-1 MemberStorageManager mgs2/ls_test
  387. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  388. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  389. cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
  390. TestClusterMember-2 MemberStorageManager mgs3/ls_test
  391. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  392. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  393. cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
  394. TestClusterMember-3 MemberStorageManager mgs4/ls_test
  395. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  396. cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  397. `[1:] && res != `
  398. TestClusterMember-0 MemberStorageManager mgs1/ls_test
  399. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  400. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  401. TestClusterMember-1 MemberStorageManager mgs2/ls_test
  402. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  403. cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  404. cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
  405. TestClusterMember-2 MemberStorageManager mgs3/ls_test
  406. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  407. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  408. cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
  409. TestClusterMember-3 MemberStorageManager mgs4/ls_test
  410. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  411. cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  412. `[1:] && res != `
  413. TestClusterMember-0 MemberStorageManager mgs1/ls_test
  414. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  415. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  416. TestClusterMember-1 MemberStorageManager mgs2/ls_test
  417. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  418. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  419. cloc: 2 (v:1) - lloc: 2 - "\b\f\x00\x05test2"
  420. TestClusterMember-2 MemberStorageManager mgs3/ls_test
  421. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  422. cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  423. cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
  424. TestClusterMember-3 MemberStorageManager mgs4/ls_test
  425. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  426. cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  427. `[1:] && res != `
  428. TestClusterMember-0 MemberStorageManager mgs1/ls_test
  429. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  430. cloc: 0 (v:1) - lloc: 1 - "\b\f\x00\x05test1"
  431. TestClusterMember-1 MemberStorageManager mgs2/ls_test
  432. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  433. cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  434. cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
  435. TestClusterMember-2 MemberStorageManager mgs3/ls_test
  436. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  437. cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  438. cloc: 0 (v:1) - lloc: 2 - "\b\f\x00\x05test1"
  439. TestClusterMember-3 MemberStorageManager mgs4/ls_test
  440. Roots: 0=0 1=0 2=0 3=0 4=0 5=0 6=0 7=0 8=0 9=0
  441. cloc: 2 (v:1) - lloc: 1 - "\b\f\x00\x05test2"
  442. `[1:] {
  443. t.Error("Unexpected cluster storage layout: ", res)
  444. return
  445. }
  446. }