manager_test.go 42 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189
  1. /*
  2. * EliasDB
  3. *
  4. * Copyright 2016 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the Mozilla Public
  7. * License, v. 2.0. If a copy of the MPL was not distributed with this
  8. * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  9. */
  10. package manager
  11. import (
  12. "bytes"
  13. "encoding/gob"
  14. "encoding/json"
  15. "errors"
  16. "flag"
  17. "fmt"
  18. "io"
  19. "io/ioutil"
  20. "log"
  21. "os"
  22. "strings"
  23. "testing"
  24. "time"
  25. )
  26. var consoleOutput = false
  27. var liveOutput = false
  28. type LogWriter struct {
  29. w io.Writer
  30. }
  31. func (l LogWriter) Write(p []byte) (n int, err error) {
  32. if liveOutput {
  33. fmt.Print(string(p))
  34. }
  35. return l.w.Write(p)
  36. }
  37. func TestMain(m *testing.M) {
  38. flag.Parse()
  39. // Create output capture file
  40. outFile, err := os.Create("out.txt")
  41. if err != nil {
  42. panic(err)
  43. }
  44. // Ensure logging is directed to the file
  45. log.SetOutput(LogWriter{outFile})
  46. // Create memberErrors map
  47. MemberErrors = make(map[string]error)
  48. MemberErrorExceptions = make(map[string][]string)
  49. // Disable housekeeping by default
  50. runHousekeeping = false
  51. defer func() { runHousekeeping = true }()
  52. // Run the tests
  53. res := m.Run()
  54. log.SetOutput(os.Stderr)
  55. // Collected output
  56. outFile.Sync()
  57. outFile.Close()
  58. stdout, err := ioutil.ReadFile("out.txt")
  59. if err != nil {
  60. panic(err)
  61. }
  62. // Handle collected output
  63. if consoleOutput {
  64. fmt.Println(string(stdout))
  65. }
  66. os.RemoveAll("out.txt")
  67. os.Exit(res)
  68. }
  69. /*
  70. Create a cluster with n members (all storage is in memory)
  71. */
  72. func createCluster(n int) []*MemberManager {
  73. var mms []*MemberManager
  74. for i := 0; i < n; i++ {
  75. mm := NewMemberManager(fmt.Sprintf("localhost:%v", 9020+i),
  76. fmt.Sprintf("TestClusterMember-%v", i), "test123", NewMemStateInfo())
  77. mm.SetEventHandler(func() {}, func() {})
  78. mms = append(mms, mm)
  79. }
  80. return mms
  81. }
  82. // Test network failure
  83. type testNetError struct {
  84. }
  85. func (*testNetError) Error() string {
  86. return "test.net.Error"
  87. }
  88. func (*testNetError) Timeout() bool {
  89. return false
  90. }
  91. func (*testNetError) Temporary() bool {
  92. return true
  93. }
  94. type testDataReq struct {
  95. Test1 string
  96. Test2 interface{}
  97. Test3 map[string]interface{}
  98. }
  99. func TestDataRequest(t *testing.T) {
  100. // Debug logging
  101. // liveOutput = true
  102. // LogDebug = LogInfo
  103. // defer func() { liveOutput = false }()
  104. cluster2 := createCluster(2)
  105. cluster2[0].Start()
  106. cluster2[1].Start()
  107. defer cluster2[0].Shutdown()
  108. defer cluster2[1].Shutdown()
  109. // Join up the cluster
  110. cluster2[0].JoinCluster(cluster2[1].name, cluster2[1].Client.rpc)
  111. // Register test data request with gob
  112. gob.Register(&testDataReq{})
  113. // Register handler on one member
  114. var res *testDataReq
  115. testdata := &testDataReq{"123", []byte{1, 2, 3}, map[string]interface{}{
  116. "test1": 1.012,
  117. "test2": true,
  118. "test3": []string{"a", "b"},
  119. }}
  120. // Check that nothing goes wrong if no handler is installed
  121. reqres, err := cluster2[0].Client.SendDataRequest(cluster2[1].name, testdata)
  122. if err != nil || reqres != nil {
  123. t.Error(err)
  124. return
  125. }
  126. cluster2[1].SetHandleDataRequest(func(data interface{}, response *interface{}) error {
  127. res = data.(*testDataReq)
  128. *response = "testok"
  129. return nil
  130. })
  131. reqres, err = cluster2[0].Client.SendDataRequest(cluster2[1].name, testdata)
  132. if err != nil {
  133. t.Error(err)
  134. return
  135. } else if reqres != "testok" {
  136. t.Error("Unexpected request response:", reqres)
  137. return
  138. } else if res.Test1 != testdata.Test1 ||
  139. fmt.Sprint(res.Test2) != fmt.Sprint(testdata.Test2) ||
  140. fmt.Sprint(res.Test3["test1"]) != fmt.Sprint(testdata.Test3["test1"]) ||
  141. fmt.Sprint(res.Test3["test2"]) != fmt.Sprint(testdata.Test3["test2"]) ||
  142. fmt.Sprint(res.Test3["test3"]) != fmt.Sprint(testdata.Test3["test3"]) {
  143. t.Error("Data got changed while in transfer:", res, testdata)
  144. return
  145. }
  146. // Test error return
  147. cluster2[1].SetHandleDataRequest(func(data interface{}, response *interface{}) error {
  148. return errors.New("TestError")
  149. })
  150. _, err = cluster2[0].Client.SendDataRequest(cluster2[1].name, testdata)
  151. if err.Error() != "ClusterError: Member error (TestError)" {
  152. t.Error(err)
  153. return
  154. }
  155. }
  156. func TestCluster2MemberCluster(t *testing.T) {
  157. // Debug logging
  158. // liveOutput = true
  159. // LogDebug = LogInfo
  160. // defer func() { liveOutput = false }()
  161. cluster2 := createCluster(2)
  162. cluster2[0].Start()
  163. cluster2[1].Start()
  164. defer cluster2[0].Shutdown()
  165. defer cluster2[1].Shutdown()
  166. // Join up the cluster
  167. cluster2[0].JoinCluster(cluster2[1].name, cluster2[1].Client.rpc)
  168. // Check state info
  169. if err := checkStateInfo(cluster2[0], `
  170. {
  171. "failed": null,
  172. "members": [
  173. "TestClusterMember-0",
  174. "localhost:9020",
  175. "TestClusterMember-1",
  176. "localhost:9021"
  177. ],
  178. "replication": 1,
  179. "ts": [
  180. "TestClusterMember-1",
  181. "2"
  182. ],
  183. "tsold": [
  184. "TestClusterMember-1",
  185. "1"
  186. ]
  187. }
  188. `[1:]); err != nil {
  189. t.Error(err)
  190. return
  191. } else if err := checkStateInfo(cluster2[1], `
  192. {
  193. "failed": null,
  194. "members": [
  195. "TestClusterMember-1",
  196. "localhost:9021",
  197. "TestClusterMember-0",
  198. "localhost:9020"
  199. ],
  200. "replication": 1,
  201. "ts": [
  202. "TestClusterMember-1",
  203. "2"
  204. ],
  205. "tsold": [
  206. "TestClusterMember-1",
  207. "1"
  208. ]
  209. }
  210. `[1:]); err != nil {
  211. t.Error(err)
  212. return
  213. }
  214. // Break up the cluster - let a member eject itself
  215. cluster2[0].EjectMember(cluster2[0].name)
  216. if err := checkStateInfo(cluster2[0], `
  217. {
  218. "failed": null,
  219. "members": [
  220. "TestClusterMember-0",
  221. "localhost:9020"
  222. ],
  223. "replication": 1,
  224. "ts": [
  225. "TestClusterMember-0",
  226. "3"
  227. ],
  228. "tsold": [
  229. "TestClusterMember-1",
  230. "2"
  231. ]
  232. }
  233. `[1:]); err != nil {
  234. t.Error(err)
  235. return
  236. } else if err := checkStateInfo(cluster2[1], `
  237. {
  238. "failed": null,
  239. "members": [
  240. "TestClusterMember-1",
  241. "localhost:9021"
  242. ],
  243. "replication": 1,
  244. "ts": [
  245. "TestClusterMember-1",
  246. "3"
  247. ],
  248. "tsold": [
  249. "TestClusterMember-1",
  250. "2"
  251. ]
  252. }
  253. `[1:]); err != nil {
  254. t.Error(err)
  255. return
  256. }
  257. // Join up the cluster again
  258. cluster2[1].JoinCluster(cluster2[0].name, cluster2[0].Client.rpc)
  259. if err := checkStateInfo(cluster2[0], `
  260. {
  261. "failed": null,
  262. "members": [
  263. "TestClusterMember-0",
  264. "localhost:9020",
  265. "TestClusterMember-1",
  266. "localhost:9021"
  267. ],
  268. "replication": 1,
  269. "ts": [
  270. "TestClusterMember-0",
  271. "4"
  272. ],
  273. "tsold": [
  274. "TestClusterMember-0",
  275. "3"
  276. ]
  277. }
  278. `[1:]); err != nil {
  279. t.Error(err)
  280. return
  281. } else if err := checkStateInfo(cluster2[1], `
  282. {
  283. "failed": null,
  284. "members": [
  285. "TestClusterMember-1",
  286. "localhost:9021",
  287. "TestClusterMember-0",
  288. "localhost:9020"
  289. ],
  290. "replication": 1,
  291. "ts": [
  292. "TestClusterMember-0",
  293. "4"
  294. ],
  295. "tsold": [
  296. "TestClusterMember-0",
  297. "3"
  298. ]
  299. }
  300. `[1:]); err != nil {
  301. t.Error(err)
  302. return
  303. }
  304. // Break up the cluster - eject the other member
  305. // the state on the other member is not updated
  306. cluster2[0].EjectMember(cluster2[1].name)
  307. if err := checkStateInfo(cluster2[0], `
  308. {
  309. "failed": null,
  310. "members": [
  311. "TestClusterMember-0",
  312. "localhost:9020"
  313. ],
  314. "replication": 1,
  315. "ts": [
  316. "TestClusterMember-0",
  317. "5"
  318. ],
  319. "tsold": [
  320. "TestClusterMember-0",
  321. "4"
  322. ]
  323. }
  324. `[1:]); err != nil {
  325. t.Error(err)
  326. return
  327. } else if err := checkStateInfo(cluster2[1], `
  328. {
  329. "failed": null,
  330. "members": [
  331. "TestClusterMember-1",
  332. "localhost:9021",
  333. "TestClusterMember-0",
  334. "localhost:9020"
  335. ],
  336. "replication": 1,
  337. "ts": [
  338. "TestClusterMember-0",
  339. "4"
  340. ],
  341. "tsold": [
  342. "TestClusterMember-0",
  343. "3"
  344. ]
  345. }
  346. `[1:]); err != nil {
  347. t.Error(err)
  348. return
  349. }
  350. // Try to rejoin from a member of the cluster - the member which did not update
  351. // its state should decline this as it thinks it is still part of the cluster
  352. cluster2[0].JoinCluster(cluster2[1].name, cluster2[1].Client.rpc)
  353. if err := checkStateInfo(cluster2[0], `
  354. {
  355. "failed": null,
  356. "members": [
  357. "TestClusterMember-0",
  358. "localhost:9020"
  359. ],
  360. "replication": 1,
  361. "ts": [
  362. "TestClusterMember-0",
  363. "5"
  364. ],
  365. "tsold": [
  366. "TestClusterMember-0",
  367. "4"
  368. ]
  369. }
  370. `[1:]); err != nil {
  371. t.Error(err)
  372. return
  373. } else if err := checkStateInfo(cluster2[1], `
  374. {
  375. "failed": null,
  376. "members": [
  377. "TestClusterMember-1",
  378. "localhost:9021",
  379. "TestClusterMember-0",
  380. "localhost:9020"
  381. ],
  382. "replication": 1,
  383. "ts": [
  384. "TestClusterMember-0",
  385. "4"
  386. ],
  387. "tsold": [
  388. "TestClusterMember-0",
  389. "3"
  390. ]
  391. }
  392. `[1:]); err != nil {
  393. t.Error(err)
  394. return
  395. }
  396. // Join up the cluster one last time from the member which did not update its
  397. // state - all should be well afterwards ...
  398. cluster2[1].JoinCluster(cluster2[0].name, cluster2[0].Client.rpc)
  399. if err := checkStateInfo(cluster2[0], `
  400. {
  401. "failed": null,
  402. "members": [
  403. "TestClusterMember-0",
  404. "localhost:9020",
  405. "TestClusterMember-1",
  406. "localhost:9021"
  407. ],
  408. "replication": 1,
  409. "ts": [
  410. "TestClusterMember-0",
  411. "6"
  412. ],
  413. "tsold": [
  414. "TestClusterMember-0",
  415. "5"
  416. ]
  417. }
  418. `[1:]); err != nil {
  419. t.Error(err)
  420. return
  421. } else if err := checkStateInfo(cluster2[1], `
  422. {
  423. "failed": null,
  424. "members": [
  425. "TestClusterMember-1",
  426. "localhost:9021",
  427. "TestClusterMember-0",
  428. "localhost:9020"
  429. ],
  430. "replication": 1,
  431. "ts": [
  432. "TestClusterMember-0",
  433. "6"
  434. ],
  435. "tsold": [
  436. "TestClusterMember-0",
  437. "5"
  438. ]
  439. }
  440. `[1:]); err != nil {
  441. t.Error(err)
  442. return
  443. }
  444. }
  445. func TestMemberInfo(t *testing.T) {
  446. cluster3 := createCluster(3)
  447. for i, member := range cluster3 {
  448. err := member.Start()
  449. defer member.Shutdown()
  450. if err != nil {
  451. t.Error(err)
  452. return
  453. }
  454. if i > 0 {
  455. // Join up the cluster - members 1, 2 join member 0
  456. if err := member.JoinCluster(cluster3[0].name,
  457. cluster3[0].Client.rpc); err != nil {
  458. t.Error(err)
  459. return
  460. }
  461. }
  462. }
  463. // Simulate failure of member 2
  464. MemberErrors[cluster3[2].name] = &testNetError{}
  465. // Reset error maps
  466. defer func() {
  467. MemberErrors = make(map[string]error)
  468. }()
  469. // Set member info on the members
  470. cluster3[0].MemberInfo()["123"] = "v123"
  471. cluster3[1].MemberInfo()["456"] = "v456"
  472. cluster3[2].MemberInfo()["789"] = "v789"
  473. // Request all member infos
  474. mi := cluster3[0].MemberInfoCluster()
  475. var w bytes.Buffer
  476. ret := json.NewEncoder(&w)
  477. ret.Encode(mi)
  478. out := bytes.Buffer{}
  479. err := json.Indent(&out, w.Bytes(), "", " ")
  480. if err != nil {
  481. t.Error(err)
  482. return
  483. }
  484. expectedClusterMemberInfo := `
  485. {
  486. "TestClusterMember-0": {
  487. "123": "v123"
  488. },
  489. "TestClusterMember-1": {
  490. "456": "v456"
  491. },
  492. "TestClusterMember-2": {
  493. "error": "ClusterError: Network error (test.net.Error)"
  494. }
  495. }
  496. `[1:]
  497. if out.String() != expectedClusterMemberInfo {
  498. t.Errorf("Unexpected cluster member info: %v\nexpected: %v",
  499. out.String(), expectedClusterMemberInfo)
  500. }
  501. }
  502. func TestClusterHouseKeeping(t *testing.T) {
  503. var log []string
  504. origLogDebug := LogDebug
  505. LogDebug = func(v ...interface{}) {
  506. log = append(log, fmt.Sprint(v...))
  507. }
  508. defer func() {
  509. LogDebug = origLogDebug
  510. }()
  511. c := createCluster(1)[0]
  512. // Activate housekeeping for this test
  513. oldRunHousekeeping := runHousekeeping
  514. oldFreqHousekeeping := FreqHousekeeping
  515. runHousekeeping = true
  516. logHousekeeping = true
  517. FreqHousekeeping = 10
  518. defer func() {
  519. runHousekeeping = oldRunHousekeeping
  520. FreqHousekeeping = oldFreqHousekeeping
  521. logHousekeeping = false
  522. }()
  523. c.Start()
  524. time.Sleep(60 * time.Millisecond)
  525. c.Shutdown()
  526. hkCount := 0
  527. for _, l := range log {
  528. if strings.Contains(l, "(HK): Running housekeeping task") {
  529. hkCount++
  530. }
  531. }
  532. if hkCount < 3 {
  533. t.Error("Unexpected count of housekeeping thread runs:", hkCount)
  534. }
  535. // Test shutting down a member twice
  536. if err := c.Shutdown(); err != nil {
  537. t.Error("Unexpected result", err)
  538. return
  539. } else if log[len(log)-1] != "Member manager TestClusterMember-0 already shut down" {
  540. t.Error("Unexpected result", err)
  541. return
  542. }
  543. }
  544. func TestClusterEjection(t *testing.T) {
  545. var err error
  546. cluster3 := createCluster(3)
  547. for i, member := range cluster3 {
  548. err := member.Start()
  549. defer member.Shutdown()
  550. if err != nil {
  551. t.Error(err)
  552. return
  553. }
  554. if i > 0 {
  555. // Join up the cluster - members 1, 2 join member 0
  556. if err := member.JoinCluster(cluster3[0].name,
  557. cluster3[0].Client.rpc); err != nil {
  558. t.Error(err)
  559. return
  560. }
  561. }
  562. }
  563. // Debug logging
  564. // liveOutput = true
  565. // LogDebug = LogInfo
  566. // defer func() { liveOutput = false }()
  567. // Try to double join a member
  568. err = cluster3[1].JoinNewMember(cluster3[2].Name(), cluster3[2].Client.rpc)
  569. if err.Error() != "ClusterError: Cluster configuration error (Cannot add member TestClusterMember-2 as a member with the same name exists already)" {
  570. t.Error("Unexpected result:", err)
  571. return
  572. }
  573. if err := checkStateInfo(cluster3[2], `
  574. {
  575. "failed": null,
  576. "members": [
  577. "TestClusterMember-2",
  578. "localhost:9022",
  579. "TestClusterMember-0",
  580. "localhost:9020",
  581. "TestClusterMember-1",
  582. "localhost:9021"
  583. ],
  584. "replication": 1,
  585. "ts": [
  586. "TestClusterMember-0",
  587. "3"
  588. ],
  589. "tsold": [
  590. "TestClusterMember-0",
  591. "2"
  592. ]
  593. }
  594. `[1:]); err != nil {
  595. t.Error(err)
  596. return
  597. }
  598. // Simulate failure of member 2
  599. MemberErrors[cluster3[2].name] = &testNetError{}
  600. // Reset error maps
  601. defer func() {
  602. MemberErrors = make(map[string]error)
  603. }()
  604. cluster3[0].StopHousekeeping = true
  605. cluster3[0].HousekeepingWorker()
  606. cluster3[0].StopHousekeeping = false
  607. cluster3[0].HousekeepingWorker()
  608. if fp := fmt.Sprint(cluster3[0].Client.FailedPeers()); fp != "[TestClusterMember-2]" {
  609. t.Error("Unexpected result:", fp)
  610. return
  611. }
  612. if err := checkStateInfo(cluster3[0], `
  613. {
  614. "failed": [
  615. "TestClusterMember-2",
  616. "test.net.Error"
  617. ],
  618. "members": [
  619. "TestClusterMember-0",
  620. "localhost:9020",
  621. "TestClusterMember-1",
  622. "localhost:9021",
  623. "TestClusterMember-2",
  624. "localhost:9022"
  625. ],
  626. "replication": 1,
  627. "ts": [
  628. "TestClusterMember-0",
  629. "4"
  630. ],
  631. "tsold": [
  632. "TestClusterMember-0",
  633. "3"
  634. ]
  635. }
  636. `[1:]); err != nil {
  637. t.Error(err)
  638. return
  639. } else if err := checkStateInfo(cluster3[2], `
  640. {
  641. "failed": null,
  642. "members": [
  643. "TestClusterMember-2",
  644. "localhost:9022",
  645. "TestClusterMember-0",
  646. "localhost:9020",
  647. "TestClusterMember-1",
  648. "localhost:9021"
  649. ],
  650. "replication": 1,
  651. "ts": [
  652. "TestClusterMember-0",
  653. "3"
  654. ],
  655. "tsold": [
  656. "TestClusterMember-0",
  657. "2"
  658. ]
  659. }
  660. `[1:]); err != nil {
  661. t.Error(err)
  662. return
  663. }
  664. // Now eject member 2 from the cluster via member 1
  665. if err := cluster3[0].Client.SendEjectMember(
  666. cluster3[1].name, cluster3[2].name); err != nil {
  667. t.Error(err)
  668. return
  669. }
  670. if err := checkStateInfo(cluster3[0], `
  671. {
  672. "failed": null,
  673. "members": [
  674. "TestClusterMember-0",
  675. "localhost:9020",
  676. "TestClusterMember-1",
  677. "localhost:9021"
  678. ],
  679. "replication": 1,
  680. "ts": [
  681. "TestClusterMember-1",
  682. "5"
  683. ],
  684. "tsold": [
  685. "TestClusterMember-0",
  686. "4"
  687. ]
  688. }
  689. `[1:]); err != nil {
  690. t.Error(err)
  691. return
  692. }
  693. // Now member 2 comes back
  694. MemberErrors = make(map[string]error)
  695. // Requests which require cluster membership should now fail
  696. err = cluster3[2].Client.SendAcquireClusterLock("123")
  697. if err.Error() != "ClusterError: Member error (Client is not a cluster member)" {
  698. t.Error(err)
  699. return
  700. }
  701. // Member detect that it was ejected
  702. cluster3[2].HousekeepingWorker()
  703. if err := checkStateInfo(cluster3[2], `
  704. {
  705. "failed": [
  706. "TestClusterMember-0",
  707. "Client is not a cluster member",
  708. "TestClusterMember-1",
  709. "Client is not a cluster member"
  710. ],
  711. "members": [
  712. "TestClusterMember-2",
  713. "localhost:9022",
  714. "TestClusterMember-0",
  715. "localhost:9020",
  716. "TestClusterMember-1",
  717. "localhost:9021"
  718. ],
  719. "replication": 1,
  720. "ts": [
  721. "TestClusterMember-2",
  722. "4"
  723. ],
  724. "tsold": [
  725. "TestClusterMember-0",
  726. "3"
  727. ]
  728. }
  729. `[1:]); err != nil {
  730. t.Error(err)
  731. return
  732. }
  733. // Join member 2 again
  734. if err := cluster3[2].JoinCluster(cluster3[0].name,
  735. cluster3[0].Client.rpc); err != nil {
  736. t.Error(err)
  737. return
  738. }
  739. cluster3[2].HousekeepingWorker()
  740. if err := checkStateInfo(cluster3[2], `
  741. {
  742. "failed": null,
  743. "members": [
  744. "TestClusterMember-2",
  745. "localhost:9022",
  746. "TestClusterMember-0",
  747. "localhost:9020",
  748. "TestClusterMember-1",
  749. "localhost:9021"
  750. ],
  751. "replication": 1,
  752. "ts": [
  753. "TestClusterMember-0",
  754. "6"
  755. ],
  756. "tsold": [
  757. "TestClusterMember-1",
  758. "5"
  759. ]
  760. }
  761. `[1:]); err != nil {
  762. t.Error(err)
  763. return
  764. } else if err := checkStateInfo(cluster3[1], `
  765. {
  766. "failed": null,
  767. "members": [
  768. "TestClusterMember-1",
  769. "localhost:9021",
  770. "TestClusterMember-0",
  771. "localhost:9020",
  772. "TestClusterMember-2",
  773. "localhost:9022"
  774. ],
  775. "replication": 1,
  776. "ts": [
  777. "TestClusterMember-0",
  778. "6"
  779. ],
  780. "tsold": [
  781. "TestClusterMember-1",
  782. "5"
  783. ]
  784. }
  785. `[1:]); err != nil {
  786. t.Error(err)
  787. return
  788. }
  789. }
  790. func TestClusterTemporaryFailure(t *testing.T) {
  791. cluster4 := createCluster(5)
  792. // Start and join the members and ensure they are shut down after the test finishes
  793. for i, member := range cluster4 {
  794. err := member.Start()
  795. defer member.Shutdown()
  796. if err != nil {
  797. t.Error(err)
  798. return
  799. }
  800. if i > 0 && i < 4 {
  801. // Join up the cluster - members 1, 2, 3 join member 0 - member 4 stays on its own
  802. if err := member.JoinCluster(cluster4[0].name,
  803. cluster4[0].Client.rpc); err != nil {
  804. t.Error(err)
  805. return
  806. }
  807. }
  808. }
  809. // Debug logging
  810. // liveOutput = true
  811. // LogDebug = LogInfo
  812. // defer func() { liveOutput = false }()
  813. // Simulate network partitioning (Member 0 and 1 can talk and
  814. // member 2, 3 and 4 can talk)
  815. MemberErrors[cluster4[0].name] = &testNetError{}
  816. MemberErrors[cluster4[1].name] = &testNetError{}
  817. MemberErrorExceptions[cluster4[0].name] = []string{cluster4[1].name}
  818. MemberErrorExceptions[cluster4[1].name] = []string{cluster4[0].name}
  819. MemberErrors[cluster4[2].name] = &testNetError{}
  820. MemberErrors[cluster4[3].name] = &testNetError{}
  821. MemberErrorExceptions[cluster4[2].name] = []string{cluster4[3].name, cluster4[4].name}
  822. MemberErrorExceptions[cluster4[3].name] = []string{cluster4[2].name, cluster4[4].name}
  823. MemberErrorExceptions[cluster4[4].name] = []string{cluster4[2].name, cluster4[3].name}
  824. // Reset error maps
  825. defer func() {
  826. MemberErrors = make(map[string]error)
  827. MemberErrorExceptions = make(map[string][]string)
  828. }()
  829. if err := checkStateInfo(cluster4[1], `
  830. {
  831. "failed": null,
  832. "members": [
  833. "TestClusterMember-1",
  834. "localhost:9021",
  835. "TestClusterMember-0",
  836. "localhost:9020",
  837. "TestClusterMember-2",
  838. "localhost:9022",
  839. "TestClusterMember-3",
  840. "localhost:9023"
  841. ],
  842. "replication": 1,
  843. "ts": [
  844. "TestClusterMember-0",
  845. "4"
  846. ],
  847. "tsold": [
  848. "TestClusterMember-0",
  849. "3"
  850. ]
  851. }
  852. `[1:]); err != nil {
  853. t.Error(err)
  854. return
  855. } else if err := checkStateInfo(cluster4[2], `
  856. {
  857. "failed": null,
  858. "members": [
  859. "TestClusterMember-2",
  860. "localhost:9022",
  861. "TestClusterMember-0",
  862. "localhost:9020",
  863. "TestClusterMember-1",
  864. "localhost:9021",
  865. "TestClusterMember-3",
  866. "localhost:9023"
  867. ],
  868. "replication": 1,
  869. "ts": [
  870. "TestClusterMember-0",
  871. "4"
  872. ],
  873. "tsold": [
  874. "TestClusterMember-0",
  875. "3"
  876. ]
  877. }
  878. `[1:]); err != nil {
  879. t.Error(err)
  880. return
  881. }
  882. // Simulate housekeeping on all members
  883. cluster4[0].HousekeepingWorker()
  884. cluster4[1].HousekeepingWorker()
  885. cluster4[2].HousekeepingWorker()
  886. cluster4[3].HousekeepingWorker()
  887. // Send invalid add new member from a (simulated) pure client
  888. rpcbak := cluster4[4].Client.rpc
  889. cluster4[4].Client.rpc = ""
  890. err := cluster4[4].JoinCluster(cluster4[3].name, "")
  891. if err.Error() != "ClusterError: Cluster configuration error (Cannot add member without RPC interface)" {
  892. t.Error(err)
  893. return
  894. }
  895. cluster4[4].Client.rpc = rpcbak
  896. // Add a new member
  897. if err := cluster4[4].JoinCluster(cluster4[3].name,
  898. cluster4[3].Client.rpc); err != nil {
  899. t.Error(err)
  900. return
  901. }
  902. // Check lists
  903. if ml := fmt.Sprint(cluster4[0].Members()); ml != "[TestClusterMember-0 TestClusterMember-1 TestClusterMember-2 TestClusterMember-3]" {
  904. t.Error("Unexpected members list:", ml)
  905. return
  906. }
  907. if ft := cluster4[0].Client.FailedTotal(); ft != 2 || !cluster4[0].Client.IsFailed(cluster4[2].name) || !cluster4[0].Client.IsFailed(cluster4[3].name) {
  908. t.Error("Unexpected failed total:", ft)
  909. return
  910. }
  911. // Member 0 and 1 think that member 2 and 3 are not reachable and vice versa
  912. // There is now a conflicting cluster state from both network partitions
  913. if err := checkStateInfo(cluster4[0], `
  914. {
  915. "failed": [
  916. "TestClusterMember-2",
  917. "test.net.Error",
  918. "TestClusterMember-3",
  919. "test.net.Error"
  920. ],
  921. "members": [
  922. "TestClusterMember-0",
  923. "localhost:9020",
  924. "TestClusterMember-1",
  925. "localhost:9021",
  926. "TestClusterMember-2",
  927. "localhost:9022",
  928. "TestClusterMember-3",
  929. "localhost:9023"
  930. ],
  931. "replication": 1,
  932. "ts": [
  933. "TestClusterMember-0",
  934. "5"
  935. ],
  936. "tsold": [
  937. "TestClusterMember-0",
  938. "4"
  939. ]
  940. }
  941. `[1:]); err != nil {
  942. t.Error(err)
  943. return
  944. } else if err := checkStateInfo(cluster4[1], `
  945. {
  946. "failed": [
  947. "TestClusterMember-2",
  948. "test.net.Error",
  949. "TestClusterMember-3",
  950. "test.net.Error"
  951. ],
  952. "members": [
  953. "TestClusterMember-1",
  954. "localhost:9021",
  955. "TestClusterMember-0",
  956. "localhost:9020",
  957. "TestClusterMember-2",
  958. "localhost:9022",
  959. "TestClusterMember-3",
  960. "localhost:9023"
  961. ],
  962. "replication": 1,
  963. "ts": [
  964. "TestClusterMember-0",
  965. "5"
  966. ],
  967. "tsold": [
  968. "TestClusterMember-0",
  969. "4"
  970. ]
  971. }
  972. `[1:]); err != nil {
  973. t.Error(err)
  974. return
  975. } else if err := checkStateInfo(cluster4[2], `
  976. {
  977. "failed": [
  978. "TestClusterMember-0",
  979. "test.net.Error",
  980. "TestClusterMember-1",
  981. "test.net.Error"
  982. ],
  983. "members": [
  984. "TestClusterMember-2",
  985. "localhost:9022",
  986. "TestClusterMember-0",
  987. "localhost:9020",
  988. "TestClusterMember-1",
  989. "localhost:9021",
  990. "TestClusterMember-3",
  991. "localhost:9023",
  992. "TestClusterMember-4",
  993. "localhost:9024"
  994. ],
  995. "replication": 1,
  996. "ts": [
  997. "TestClusterMember-3",
  998. "6"
  999. ],
  1000. "tsold": [
  1001. "TestClusterMember-2",
  1002. "5"
  1003. ]
  1004. }
  1005. `[1:]); err != nil {
  1006. t.Error(err)
  1007. return
  1008. } else if err := checkStateInfo(cluster4[3], `
  1009. {
  1010. "failed": [
  1011. "TestClusterMember-0",
  1012. "test.net.Error",
  1013. "TestClusterMember-1",
  1014. "test.net.Error"
  1015. ],
  1016. "members": [
  1017. "TestClusterMember-3",
  1018. "localhost:9023",
  1019. "TestClusterMember-0",
  1020. "localhost:9020",
  1021. "TestClusterMember-1",
  1022. "localhost:9021",
  1023. "TestClusterMember-2",
  1024. "localhost:9022",
  1025. "TestClusterMember-4",
  1026. "localhost:9024"
  1027. ],
  1028. "replication": 1,
  1029. "ts": [
  1030. "TestClusterMember-3",
  1031. "6"
  1032. ],
  1033. "tsold": [
  1034. "TestClusterMember-2",
  1035. "5"
  1036. ]
  1037. }
  1038. `[1:]); err != nil {
  1039. t.Error(err)
  1040. return
  1041. }
  1042. // Remove the network partitions
  1043. MemberErrors = make(map[string]error)
  1044. MemberErrorExceptions = make(map[string][]string)
  1045. // Simulate housekeeping on member 0 kicks in first
  1046. cluster4[0].HousekeepingWorker()
  1047. if err := checkStateInfo(cluster4[0], `
  1048. {
  1049. "failed": null,
  1050. "members": [
  1051. "TestClusterMember-0",
  1052. "localhost:9020",
  1053. "TestClusterMember-1",
  1054. "localhost:9021",
  1055. "TestClusterMember-2",
  1056. "localhost:9022",
  1057. "TestClusterMember-3",
  1058. "localhost:9023",
  1059. "TestClusterMember-4",
  1060. "localhost:9024"
  1061. ],
  1062. "replication": 1,
  1063. "ts": [
  1064. "TestClusterMember-0",
  1065. "6"
  1066. ],
  1067. "tsold": [
  1068. "TestClusterMember-0",
  1069. "5"
  1070. ]
  1071. }
  1072. `[1:]); err != nil {
  1073. t.Error(err)
  1074. return
  1075. } else if err := checkStateInfo(cluster4[2], `
  1076. {
  1077. "failed": null,
  1078. "members": [
  1079. "TestClusterMember-2",
  1080. "localhost:9022",
  1081. "TestClusterMember-0",
  1082. "localhost:9020",
  1083. "TestClusterMember-1",
  1084. "localhost:9021",
  1085. "TestClusterMember-3",
  1086. "localhost:9023",
  1087. "TestClusterMember-4",
  1088. "localhost:9024"
  1089. ],
  1090. "replication": 1,
  1091. "ts": [
  1092. "TestClusterMember-0",
  1093. "6"
  1094. ],
  1095. "tsold": [
  1096. "TestClusterMember-0",
  1097. "5"
  1098. ]
  1099. }
  1100. `[1:]); err != nil {
  1101. t.Error(err)
  1102. return
  1103. } else if err := checkStateInfo(cluster4[4], `
  1104. {
  1105. "failed": null,
  1106. "members": [
  1107. "TestClusterMember-4",
  1108. "localhost:9024",
  1109. "TestClusterMember-0",
  1110. "localhost:9020",
  1111. "TestClusterMember-1",
  1112. "localhost:9021",
  1113. "TestClusterMember-2",
  1114. "localhost:9022",
  1115. "TestClusterMember-3",
  1116. "localhost:9023"
  1117. ],
  1118. "replication": 1,
  1119. "ts": [
  1120. "TestClusterMember-0",
  1121. "6"
  1122. ],
  1123. "tsold": [
  1124. "TestClusterMember-0",
  1125. "5"
  1126. ]
  1127. }
  1128. `[1:]); err != nil {
  1129. t.Error(err)
  1130. return
  1131. }
  1132. }
  1133. func TestClusterBuilding(t *testing.T) {
  1134. cluster3 := createCluster(3)
  1135. // Start the members and ensure they are shut down after the test finishes
  1136. // Debug logging
  1137. // liveOutput = true
  1138. // LogDebug = LogInfo
  1139. for _, member := range cluster3 {
  1140. err := member.Start()
  1141. defer member.Shutdown()
  1142. if err != nil {
  1143. t.Error(err)
  1144. return
  1145. }
  1146. }
  1147. // defer func() { liveOutput = false }()
  1148. // Check state info
  1149. if err := checkStateInfo(cluster3[1], `
  1150. {
  1151. "failed": null,
  1152. "members": [
  1153. "TestClusterMember-1",
  1154. "localhost:9021"
  1155. ],
  1156. "replication": 1,
  1157. "ts": [
  1158. "TestClusterMember-1",
  1159. "1"
  1160. ],
  1161. "tsold": [
  1162. "",
  1163. "0"
  1164. ]
  1165. }
  1166. `[1:]); err != nil {
  1167. t.Error(err)
  1168. return
  1169. }
  1170. // Form the cluster by adding member 2 into the cluster of member 1
  1171. err := cluster3[2].JoinCluster(cluster3[1].name,
  1172. cluster3[1].Client.rpc)
  1173. if err != nil {
  1174. t.Error(err)
  1175. return
  1176. }
  1177. // Check state info
  1178. if err := checkStateInfo(cluster3[1], `
  1179. {
  1180. "failed": null,
  1181. "members": [
  1182. "TestClusterMember-1",
  1183. "localhost:9021",
  1184. "TestClusterMember-2",
  1185. "localhost:9022"
  1186. ],
  1187. "replication": 1,
  1188. "ts": [
  1189. "TestClusterMember-1",
  1190. "2"
  1191. ],
  1192. "tsold": [
  1193. "TestClusterMember-1",
  1194. "1"
  1195. ]
  1196. }
  1197. `[1:]); err != nil {
  1198. t.Error(err)
  1199. return
  1200. } else if err := checkStateInfo(cluster3[2], `
  1201. {
  1202. "failed": null,
  1203. "members": [
  1204. "TestClusterMember-2",
  1205. "localhost:9022",
  1206. "TestClusterMember-1",
  1207. "localhost:9021"
  1208. ],
  1209. "replication": 1,
  1210. "ts": [
  1211. "TestClusterMember-1",
  1212. "2"
  1213. ],
  1214. "tsold": [
  1215. "TestClusterMember-1",
  1216. "1"
  1217. ]
  1218. }
  1219. `[1:]); err != nil {
  1220. t.Error(err)
  1221. return
  1222. }
  1223. // Simulate member 2 becomes unavailable
  1224. MemberErrors[cluster3[2].name] = &testNetError{}
  1225. defer delete(MemberErrors, cluster3[2].name)
  1226. // Join member 0 via member 2
  1227. err = cluster3[0].JoinCluster(cluster3[2].name,
  1228. cluster3[2].Client.rpc)
  1229. if err.Error() != "ClusterError: Network error (test.net.Error)" {
  1230. t.Error(err)
  1231. return
  1232. }
  1233. // Join member 0 via member 1
  1234. err = cluster3[0].JoinCluster(cluster3[1].name,
  1235. cluster3[1].Client.rpc)
  1236. if err != nil {
  1237. t.Error(err)
  1238. return
  1239. }
  1240. // Check state info - Member 1 knows now that member 2 has failed
  1241. if err := checkStateInfo(cluster3[1], `
  1242. {
  1243. "failed": [
  1244. "TestClusterMember-2",
  1245. "test.net.Error"
  1246. ],
  1247. "members": [
  1248. "TestClusterMember-1",
  1249. "localhost:9021",
  1250. "TestClusterMember-0",
  1251. "localhost:9020",
  1252. "TestClusterMember-2",
  1253. "localhost:9022"
  1254. ],
  1255. "replication": 1,
  1256. "ts": [
  1257. "TestClusterMember-1",
  1258. "3"
  1259. ],
  1260. "tsold": [
  1261. "TestClusterMember-1",
  1262. "2"
  1263. ]
  1264. }
  1265. `[1:]); err != nil {
  1266. t.Error(err)
  1267. return
  1268. } else if err := checkStateInfo(cluster3[0], `
  1269. {
  1270. "failed": [
  1271. "TestClusterMember-2",
  1272. "test.net.Error"
  1273. ],
  1274. "members": [
  1275. "TestClusterMember-0",
  1276. "localhost:9020",
  1277. "TestClusterMember-1",
  1278. "localhost:9021",
  1279. "TestClusterMember-2",
  1280. "localhost:9022"
  1281. ],
  1282. "replication": 1,
  1283. "ts": [
  1284. "TestClusterMember-1",
  1285. "3"
  1286. ],
  1287. "tsold": [
  1288. "TestClusterMember-1",
  1289. "2"
  1290. ]
  1291. }
  1292. `[1:]); err != nil {
  1293. t.Error(err)
  1294. return
  1295. }
  1296. // Simulate member 2 becomes available again
  1297. delete(MemberErrors, cluster3[2].name)
  1298. // Member 2 has still an old state info
  1299. if err := checkStateInfo(cluster3[2], `
  1300. {
  1301. "failed": null,
  1302. "members": [
  1303. "TestClusterMember-2",
  1304. "localhost:9022",
  1305. "TestClusterMember-1",
  1306. "localhost:9021"
  1307. ],
  1308. "replication": 1,
  1309. "ts": [
  1310. "TestClusterMember-1",
  1311. "2"
  1312. ],
  1313. "tsold": [
  1314. "TestClusterMember-1",
  1315. "1"
  1316. ]
  1317. }
  1318. `[1:]); err != nil {
  1319. t.Error(err)
  1320. return
  1321. }
  1322. // Member 2 should be updated the state info eventually through housekeeping
  1323. cluster3[2].HousekeepingWorker()
  1324. // Member 2 is still considered failed by the cluster
  1325. if err := checkStateInfo(cluster3[2], `
  1326. {
  1327. "failed": [
  1328. "TestClusterMember-2",
  1329. "test.net.Error"
  1330. ],
  1331. "members": [
  1332. "TestClusterMember-2",
  1333. "localhost:9022",
  1334. "TestClusterMember-0",
  1335. "localhost:9020",
  1336. "TestClusterMember-1",
  1337. "localhost:9021"
  1338. ],
  1339. "replication": 1,
  1340. "ts": [
  1341. "TestClusterMember-1",
  1342. "3"
  1343. ],
  1344. "tsold": [
  1345. "TestClusterMember-1",
  1346. "2"
  1347. ]
  1348. }
  1349. `[1:]); err != nil {
  1350. t.Error(err)
  1351. return
  1352. } else if err := checkStateInfo(cluster3[1], `
  1353. {
  1354. "failed": [
  1355. "TestClusterMember-2",
  1356. "test.net.Error"
  1357. ],
  1358. "members": [
  1359. "TestClusterMember-1",
  1360. "localhost:9021",
  1361. "TestClusterMember-0",
  1362. "localhost:9020",
  1363. "TestClusterMember-2",
  1364. "localhost:9022"
  1365. ],
  1366. "replication": 1,
  1367. "ts": [
  1368. "TestClusterMember-1",
  1369. "3"
  1370. ],
  1371. "tsold": [
  1372. "TestClusterMember-1",
  1373. "2"
  1374. ]
  1375. }
  1376. `[1:]); err != nil {
  1377. t.Error(err)
  1378. return
  1379. } else if err := checkStateInfo(cluster3[0], `
  1380. {
  1381. "failed": [
  1382. "TestClusterMember-2",
  1383. "test.net.Error"
  1384. ],
  1385. "members": [
  1386. "TestClusterMember-0",
  1387. "localhost:9020",
  1388. "TestClusterMember-1",
  1389. "localhost:9021",
  1390. "TestClusterMember-2",
  1391. "localhost:9022"
  1392. ],
  1393. "replication": 1,
  1394. "ts": [
  1395. "TestClusterMember-1",
  1396. "3"
  1397. ],
  1398. "tsold": [
  1399. "TestClusterMember-1",
  1400. "2"
  1401. ]
  1402. }
  1403. `[1:]); err != nil {
  1404. t.Error(err)
  1405. return
  1406. }
  1407. // Now housekeeping runs on member 1 which should detect that 2 is back
  1408. // again - the state info on all members should be updated
  1409. cluster3[1].HousekeepingWorker()
  1410. if err := checkStateInfo(cluster3[2], `
  1411. {
  1412. "failed": null,
  1413. "members": [
  1414. "TestClusterMember-2",
  1415. "localhost:9022",
  1416. "TestClusterMember-0",
  1417. "localhost:9020",
  1418. "TestClusterMember-1",
  1419. "localhost:9021"
  1420. ],
  1421. "replication": 1,
  1422. "ts": [
  1423. "TestClusterMember-1",
  1424. "4"
  1425. ],
  1426. "tsold": [
  1427. "TestClusterMember-1",
  1428. "3"
  1429. ]
  1430. }
  1431. `[1:]); err != nil {
  1432. t.Error(err)
  1433. return
  1434. } else if err := checkStateInfo(cluster3[1], `
  1435. {
  1436. "failed": null,
  1437. "members": [
  1438. "TestClusterMember-1",
  1439. "localhost:9021",
  1440. "TestClusterMember-0",
  1441. "localhost:9020",
  1442. "TestClusterMember-2",
  1443. "localhost:9022"
  1444. ],
  1445. "replication": 1,
  1446. "ts": [
  1447. "TestClusterMember-1",
  1448. "4"
  1449. ],
  1450. "tsold": [
  1451. "TestClusterMember-1",
  1452. "3"
  1453. ]
  1454. }
  1455. `[1:]); err != nil {
  1456. t.Error(err)
  1457. return
  1458. } else if err := checkStateInfo(cluster3[0], `
  1459. {
  1460. "failed": null,
  1461. "members": [
  1462. "TestClusterMember-0",
  1463. "localhost:9020",
  1464. "TestClusterMember-1",
  1465. "localhost:9021",
  1466. "TestClusterMember-2",
  1467. "localhost:9022"
  1468. ],
  1469. "replication": 1,
  1470. "ts": [
  1471. "TestClusterMember-1",
  1472. "4"
  1473. ],
  1474. "tsold": [
  1475. "TestClusterMember-1",
  1476. "3"
  1477. ]
  1478. }
  1479. `[1:]); err != nil {
  1480. t.Error(err)
  1481. return
  1482. }
  1483. }
  1484. func checkStateInfo(mm *MemberManager, expectedStateInfo string) error {
  1485. var w bytes.Buffer
  1486. ret := json.NewEncoder(&w)
  1487. ret.Encode(mm.stateInfo.Map())
  1488. out := bytes.Buffer{}
  1489. err := json.Indent(&out, w.Bytes(), "", " ")
  1490. if err != nil {
  1491. return err
  1492. }
  1493. if out.String() != expectedStateInfo {
  1494. return fmt.Errorf("Unexpected state info: %v\nexpected: %v",
  1495. out.String(), expectedStateInfo)
  1496. }
  1497. return nil
  1498. }
  1499. func TestStateInfo(t *testing.T) {
  1500. cluster1 := createCluster(1)
  1501. cluster1[0].Client.peers["abc"] = "localhost:123"
  1502. cluster1[0].Client.peers["def"] = "localhost:124"
  1503. cluster1[0].updateStateInfo(true)
  1504. si := cluster1[0].stateInfo
  1505. sip, _ := si.Get(StateInfoMEMBERS)
  1506. if fmt.Sprint(sip) != "[TestClusterMember-0 localhost:9020 abc localhost:123 def localhost:124]" {
  1507. t.Error("Unexpected StateInfo:", sip)
  1508. return
  1509. }
  1510. sif, _ := si.Get(StateInfoFAILED)
  1511. if fmt.Sprint(sif) != "[]" {
  1512. t.Error("Unexpected StateInfo:", sif)
  1513. return
  1514. }
  1515. cluster1[0].stateInfo = NewMemStateInfo()
  1516. cluster1[0].Client.peers = nil
  1517. cluster1[0].applyStateInfo(si.(*MemStateInfo).data)
  1518. if len(cluster1[0].stateInfo.(*MemStateInfo).data) != 5 {
  1519. t.Error("State info not correct: ", cluster1[0].stateInfo.(*MemStateInfo).data)
  1520. return
  1521. }
  1522. peers := cluster1[0].Client.peers
  1523. if len(peers) != 2 || peers["abc"] != "localhost:123" || peers["def"] != "localhost:124" {
  1524. t.Error("Unexpected peers map:", peers)
  1525. return
  1526. }
  1527. // Create a new member manager and apply a given state info
  1528. mm := NewMemberManager(fmt.Sprintf("localhost:9022"),
  1529. "TestClusterMember-9", "test123", cluster1[0].stateInfo)
  1530. if err := checkStateInfo(mm, `
  1531. {
  1532. "failed": null,
  1533. "members": [
  1534. "TestClusterMember-9",
  1535. "localhost:9022",
  1536. "TestClusterMember-0",
  1537. "localhost:9020",
  1538. "abc",
  1539. "localhost:123",
  1540. "def",
  1541. "localhost:124"
  1542. ],
  1543. "replication": 1,
  1544. "ts": [
  1545. "TestClusterMember-0",
  1546. "2"
  1547. ],
  1548. "tsold": [
  1549. "TestClusterMember-0",
  1550. "1"
  1551. ]
  1552. }
  1553. `[1:]); err != nil {
  1554. t.Error(err)
  1555. return
  1556. }
  1557. }
  1558. func TestLowLevelManagerCommunication(t *testing.T) {
  1559. cluster3 := createCluster(3)
  1560. // Try starting with an invalid rpc
  1561. origRPC := cluster3[0].Client.rpc
  1562. cluster3[0].Client.rpc = ":-1"
  1563. if err := cluster3[0].Start(); !strings.HasPrefix(err.Error(), "listen tcp") {
  1564. t.Error("Unexpected result:", err)
  1565. return
  1566. }
  1567. cluster3[0].Client.rpc = origRPC
  1568. // Start the cluster and ensure it is shut down after the test finishes
  1569. for _, member := range cluster3 {
  1570. err := member.Start()
  1571. defer member.Shutdown()
  1572. if err != nil {
  1573. t.Error(err)
  1574. return
  1575. }
  1576. }
  1577. // Check info of MemberManager
  1578. if res := cluster3[1].Name(); res != cluster3[1].name {
  1579. t.Error("Unexpected result:", res)
  1580. return
  1581. } else if res := cluster3[1].NetAddr(); res != cluster3[1].Client.rpc {
  1582. t.Error("Unexpected result:", res)
  1583. return
  1584. } else if res := cluster3[1].StateInfo(); res != cluster3[1].stateInfo {
  1585. t.Error("Unexpected result:", res)
  1586. return
  1587. }
  1588. // Do a ping which add temrorary a member
  1589. pres, err := cluster3[0].Client.SendPing(cluster3[1].Name(), cluster3[1].Client.rpc)
  1590. if err != nil || fmt.Sprint(pres) != "[Pong]" {
  1591. t.Error("Unexpected result:", pres, err)
  1592. return
  1593. }
  1594. // Manually add some peers
  1595. cluster3[0].Client.peers[cluster3[1].Name()] = cluster3[1].Client.rpc
  1596. cluster3[1].Client.peers[cluster3[1].Name()] = cluster3[1].Client.rpc
  1597. // Add invalid entry
  1598. cluster3[0].Client.peers["bla"] = "localhost:-1"
  1599. _, err = cluster3[0].Client.SendRequest("bla", RPCPing, nil)
  1600. if !strings.HasPrefix(err.Error(), "ClusterError: Network error") {
  1601. t.Error("Unexpected result:", err.Error())
  1602. return
  1603. }
  1604. // Send ping (at this point member 0 is unknown to member 1 so it is treated as a pure client)
  1605. pres, err = cluster3[0].Client.SendPing(cluster3[1].Name(), "")
  1606. if err != nil || fmt.Sprint(pres) != "[Pong]" {
  1607. t.Error("Unexpected ping result:", pres, err)
  1608. return
  1609. }
  1610. // Send ping with unknown target - fail is client side
  1611. pres, err = cluster3[0].Client.SendPing(cluster3[1].Name()+"123", "")
  1612. if err.Error() != "ClusterError: Unknown peer member (TestClusterMember-1123)" || pres != nil {
  1613. t.Error("Unexpected ping result:", pres, err)
  1614. return
  1615. }
  1616. // Send ping with unknown target - fail is server side
  1617. res, err := cluster3[0].Client.SendRequest(cluster3[1].Name(),
  1618. RPCPing, map[RequestArgument]interface{}{
  1619. RequestTARGET: cluster3[1].Name() + "123",
  1620. })
  1621. if err.Error() != "ClusterError: Member error (Unknown target member)" || res != nil {
  1622. t.Error("Unexpected ping result:", res, err)
  1623. return
  1624. }
  1625. // Send ping with invalid member token
  1626. oldAuth := cluster3[0].Client.token.MemberAuth
  1627. cluster3[0].Client.token.MemberAuth = oldAuth + "123"
  1628. pres, err = cluster3[0].Client.SendPing(cluster3[1].Name(), "")
  1629. if err.Error() != "ClusterError: Member error (Invalid member token)" || pres != nil {
  1630. t.Error("Unexpected ping result:", pres, err)
  1631. return
  1632. }
  1633. cluster3[0].Client.token.MemberAuth = oldAuth
  1634. // Test acquisition of a cluster lock
  1635. res, err = cluster3[1].Client.SendRequest(cluster3[1].Name(),
  1636. RPCAcquireLock, map[RequestArgument]interface{}{
  1637. RequestTARGET: cluster3[1].Name(),
  1638. RequestLOCK: "mylock",
  1639. })
  1640. if err != nil || res != cluster3[1].Name() {
  1641. t.Error(err, res)
  1642. return
  1643. }
  1644. // Check that the lock was set
  1645. if l := cluster3[1].Client.clusterLocks.Size(); l != 1 {
  1646. t.Error("Unexpected cluster locks structure:", l)
  1647. return
  1648. } else if l, _ := cluster3[1].Client.clusterLocks.Get("mylock"); l != cluster3[1].Name() {
  1649. t.Error("Unexpected cluster lock owner:", l)
  1650. return
  1651. }
  1652. // Try to acquire the lock for a different member
  1653. res, err = cluster3[0].Client.SendRequest(cluster3[1].Name(),
  1654. RPCAcquireLock, map[RequestArgument]interface{}{
  1655. RequestTARGET: cluster3[1].Name(),
  1656. RequestLOCK: "mylock",
  1657. })
  1658. // Check the cluster member check
  1659. if err.Error() != "ClusterError: Member error (Client is not a cluster member)" || res != nil {
  1660. t.Error(err, res)
  1661. return
  1662. }
  1663. err = cluster3[0].JoinNewMember(cluster3[1].Name(), cluster3[1].Client.rpc)
  1664. if err.Error() != "ClusterError: Member error (Client is not a cluster member)" || res != nil {
  1665. t.Error(err, res)
  1666. return
  1667. }
  1668. // Register member 0 on member 1
  1669. cluster3[1].Client.peers[cluster3[0].Name()] = cluster3[0].Client.rpc
  1670. res, err = cluster3[0].Client.SendRequest(cluster3[1].Name(),
  1671. RPCAcquireLock, map[RequestArgument]interface{}{
  1672. RequestTARGET: cluster3[1].Name(),
  1673. RequestLOCK: "mylock",
  1674. })
  1675. if err.Error() != "ClusterError: Member error (ClusterError: Requested lock is already taken (TestClusterMember-1))" || res != nil {
  1676. t.Error(err, res)
  1677. return
  1678. }
  1679. // Release a lock from a wrong member
  1680. res, err = cluster3[0].Client.SendRequest(cluster3[1].Name(),
  1681. RPCReleaseLock, map[RequestArgument]interface{}{
  1682. RequestTARGET: cluster3[1].Name(),
  1683. RequestLOCK: "mylock",
  1684. })
  1685. if err.Error() != "ClusterError: Member error (ClusterError: Requested lock not owned (Owned by TestClusterMember-1 not by TestClusterMember-0))" || res != nil {
  1686. t.Error(err, res)
  1687. return
  1688. }
  1689. // Check that the lock was not unset
  1690. if l := cluster3[1].Client.clusterLocks.Size(); l != 1 {
  1691. t.Error("Unexpected cluster locks structure:", l)
  1692. return
  1693. }
  1694. // Release the lock from the correct member
  1695. res, err = cluster3[1].Client.SendRequest(cluster3[1].Name(),
  1696. RPCReleaseLock, map[RequestArgument]interface{}{
  1697. RequestTARGET: cluster3[1].Name(),
  1698. RequestLOCK: "mylock",
  1699. })
  1700. if err != nil || res != nil {
  1701. t.Error(err, res)
  1702. return
  1703. }
  1704. // Check that the lock was unset
  1705. if l := cluster3[1].Client.clusterLocks.Size(); l != 0 {
  1706. t.Error("Unexpected cluster locks structure:", l)
  1707. return
  1708. }
  1709. // Register member 2 on member 1 and vice versa
  1710. cluster3[1].Client.peers[cluster3[2].Name()] = cluster3[2].Client.rpc
  1711. cluster3[2].Client.peers[cluster3[1].Name()] = cluster3[1].Client.rpc
  1712. // Test taking lock with serious error - member which takes the lock should release
  1713. // the ones which were already taken
  1714. MemberErrors[cluster3[2].name] = errors.New("testerror")
  1715. defer delete(MemberErrors, cluster3[2].name)
  1716. err = cluster3[1].Client.SendAcquireClusterLock("123")
  1717. if err.Error() != "ClusterError: Member error (testerror)" {
  1718. t.Error("Test error expected:", err)
  1719. return
  1720. }
  1721. // Check that the lock is not set
  1722. if l := cluster3[1].Client.clusterLocks.Size(); l != 0 {
  1723. t.Error("Unexpected cluster locks structure:", l)
  1724. return
  1725. } else if l := cluster3[0].Client.clusterLocks.Size(); l != 0 {
  1726. t.Error("Unexpected cluster locks structure:", l)
  1727. return
  1728. } else if l := cluster3[2].Client.clusterLocks.Size(); l != 0 {
  1729. t.Error("Unexpected cluster locks structure:", l)
  1730. return
  1731. }
  1732. // Check state info error
  1733. _, err = cluster3[1].Client.SendStateInfoRequest(cluster3[2].name)
  1734. if err.Error() != "ClusterError: Member error (testerror)" {
  1735. t.Error("Unexpected result:", res, err)
  1736. return
  1737. }
  1738. delete(MemberErrors, cluster3[2].name)
  1739. // Use client function to take lock
  1740. err = cluster3[0].Client.SendAcquireClusterLock("123")
  1741. if err != nil {
  1742. t.Error(err)
  1743. return
  1744. }
  1745. // Check that the lock was set
  1746. if l := cluster3[1].Client.clusterLocks.Size(); l != 1 {
  1747. t.Error("Unexpected cluster locks structure:", l)
  1748. return
  1749. } else if l := cluster3[0].Client.clusterLocks.Size(); l != 1 {
  1750. t.Error("Unexpected cluster locks structure:", l)
  1751. return
  1752. }
  1753. // Use client to unlock
  1754. err = cluster3[1].Client.SendReleaseClusterLock("123")
  1755. if err.Error() != "ClusterError: Member error (ClusterError: Requested lock not owned (Owned by TestClusterMember-0 not by TestClusterMember-1))" {
  1756. t.Error(err)
  1757. return
  1758. }
  1759. err = cluster3[0].Client.SendReleaseClusterLock("123")
  1760. if err != nil {
  1761. t.Error(err)
  1762. return
  1763. }
  1764. // Check that the lock is not set
  1765. if l := cluster3[1].Client.clusterLocks.Size(); l != 0 {
  1766. t.Error("Unexpected cluster locks structure:", l)
  1767. return
  1768. } else if l := cluster3[0].Client.clusterLocks.Size(); l != 0 {
  1769. t.Error("Unexpected cluster locks structure:", l)
  1770. return
  1771. }
  1772. // Remove wrong peer 1 entry from member 1
  1773. delete(cluster3[1].Client.peers, cluster3[1].Name())
  1774. // Acquire cluster lock for updating the state info
  1775. if err := cluster3[1].Client.SendAcquireClusterLock(ClusterLockUpdateStateInfo); err != nil {
  1776. t.Error(err)
  1777. return
  1778. }
  1779. // Try to update the state info
  1780. err = cluster3[0].UpdateClusterStateInfo()
  1781. if err.Error() != "ClusterError: Member error (ClusterError: Requested lock is already taken (TestClusterMember-1))" {
  1782. t.Error(err)
  1783. return
  1784. }
  1785. // Try to eject member 1
  1786. err = cluster3[1].EjectMember(cluster3[1].Name())
  1787. if err.Error() != "ClusterError: Member error (ClusterError: Requested lock is already taken (TestClusterMember-1))" || res != nil {
  1788. t.Error(err, res)
  1789. return
  1790. }
  1791. // Release cluster lock for updating the state info
  1792. if err := cluster3[1].Client.SendReleaseClusterLock(ClusterLockUpdateStateInfo); err != nil {
  1793. t.Error(err)
  1794. return
  1795. }
  1796. // Simulate a write error while persisting the cluster state info
  1797. MsiRetFlush = errors.New("TestFlushError")
  1798. err = cluster3[1].EjectMember(cluster3[1].Name())
  1799. MsiRetFlush = nil
  1800. if err.Error() != "ClusterError: Member error (TestFlushError)" {
  1801. t.Error(err)
  1802. return
  1803. }
  1804. // Check that the lock is not left behindt
  1805. if l := cluster3[1].Client.clusterLocks.Size(); l != 0 {
  1806. t.Error("Unexpected cluster locks structure:", l)
  1807. return
  1808. } else if l := cluster3[0].Client.clusterLocks.Size(); l != 0 {
  1809. t.Error("Unexpected cluster locks structure:", l)
  1810. return
  1811. }
  1812. // Check peer is still there
  1813. _, ok := cluster3[0].Client.peers[cluster3[1].Name()]
  1814. if !ok {
  1815. t.Error("Expected member was not in peer list:", cluster3[1].Name())
  1816. return
  1817. }
  1818. // Try to update the cluster state info
  1819. MsiRetFlush = errors.New("TestFlushError")
  1820. err = cluster3[0].UpdateClusterStateInfo()
  1821. MsiRetFlush = nil
  1822. if err.Error() != "TestFlushError" {
  1823. t.Error(err)
  1824. return
  1825. }
  1826. // Check that the lock is not left behindt
  1827. if l := cluster3[1].Client.clusterLocks.Size(); l != 0 {
  1828. t.Error("Unexpected cluster locks structure:", l)
  1829. return
  1830. } else if l := cluster3[0].Client.clusterLocks.Size(); l != 0 {
  1831. t.Error("Unexpected cluster locks structure:", l)
  1832. return
  1833. }
  1834. // Actually remove the member
  1835. err = cluster3[1].EjectMember(cluster3[1].Name())
  1836. if err != nil {
  1837. t.Error(err)
  1838. return
  1839. }
  1840. _, ok = cluster3[0].Client.peers[cluster3[1].Name()]
  1841. if ok {
  1842. t.Error("Unexpected member was in peer list:", cluster3[1].Name())
  1843. return
  1844. }
  1845. }