diskstoragemanager_test.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666
  1. /*
  2. * EliasDB
  3. *
  4. * Copyright 2016 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the Mozilla Public
  7. * License, v. 2.0. If a copy of the MPL was not distributed with this
  8. * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  9. */
  10. package storage
  11. import (
  12. "bytes"
  13. "encoding/gob"
  14. "os"
  15. "sync"
  16. "testing"
  17. "time"
  18. "devt.de/krotik/common/lockutil"
  19. "devt.de/krotik/common/testutil"
  20. "devt.de/krotik/eliasdb/storage/file"
  21. "devt.de/krotik/eliasdb/storage/slotting/pageview"
  22. "devt.de/krotik/eliasdb/storage/util"
  23. )
  24. func TestDiskStorageManagerLockFilePanic(t *testing.T) {
  25. dsm := NewDiskStorageManager(DBDIR+"/test0", false, false, true, false)
  26. defer func() {
  27. if r := recover(); r == nil {
  28. t.Error("Doing an operation with a storagefile with stopped lock watcher goroutine did not cause a panic.")
  29. }
  30. // Start the watcher goroutine again and close the file
  31. dsm.lockfile.Start()
  32. dsm.Close()
  33. }()
  34. file, err := os.OpenFile(DBDIR+"/test0.lck", os.O_CREATE|os.O_TRUNC, 0660)
  35. if err != nil {
  36. t.Error(err)
  37. return
  38. }
  39. file.WriteString("t")
  40. file.Close()
  41. // Give the watcher goroutine time to die
  42. time.Sleep(time.Duration(100) * time.Millisecond)
  43. if dsm.lockfile.WatcherRunning() {
  44. t.Error("Watcher goroutine did not die")
  45. }
  46. // This should cause a panic
  47. dsm.Free(util.PackLocation(2, 20))
  48. }
  49. func TestDiskStorageManager1(t *testing.T) {
  50. var res string
  51. dsm := NewDiskStorageManager(DBDIR+"/test1", false, false, true, false)
  52. if dsm.Name() != "DiskStorageFile:"+DBDIR+"/test1" {
  53. t.Error("Unexpected name for DiskStorageManager:", dsm.Name())
  54. return
  55. }
  56. // Make sure that another process which attempts to open the same
  57. // storage would panic
  58. time.Sleep(100 * time.Millisecond) // Need some time here otherwise Windows fails sometimes
  59. testLockfileStartPanic(t)
  60. // Test simple insert
  61. loc, err := dsm.Insert("This is a test")
  62. if err != nil {
  63. t.Error(err)
  64. }
  65. checkLocation(t, loc, 1, pageview.OffsetTransData)
  66. dsm.Fetch(loc, &res)
  67. if res != "This is a test" {
  68. t.Error("Unexpected fetch result:", res)
  69. }
  70. // Test case where we give a byte slice
  71. var res2 string
  72. bs := make([]byte, 18)
  73. dsm.ByteDiskStorageManager.Fetch(loc, bs)
  74. err = gob.NewDecoder(bytes.NewReader(bs)).Decode(&res2)
  75. if err != nil || string(res2) != "This is a test" {
  76. t.Error("Unexpected fetch result:", err, res2)
  77. }
  78. // Get physical slot for stored data
  79. ploc, err := dsm.logicalSlotManager.Fetch(loc)
  80. if err != nil {
  81. t.Error(err)
  82. return
  83. }
  84. // Test uodate
  85. // The next update should allocate a new physical record
  86. err = dsm.Update(loc, "This is another test")
  87. if err != nil {
  88. t.Error(err)
  89. }
  90. // Get new physical slot for stored data
  91. newPloc, err := dsm.logicalSlotManager.Fetch(loc)
  92. if err != nil {
  93. t.Error(err)
  94. return
  95. }
  96. if ploc == newPloc {
  97. t.Error("Physical address should have changed")
  98. return
  99. }
  100. dsm.Fetch(loc, &res)
  101. if res != "This is another test" {
  102. t.Error("Unexpected fetch result:", res)
  103. }
  104. // Test insert error
  105. _, err = dsm.Insert(&testutil.GobTestObject{Name: "test", EncErr: true, DecErr: false})
  106. if err == nil {
  107. t.Error(err)
  108. return
  109. }
  110. psp := dsm.physicalSlotsPager
  111. fpsp := dsm.physicalFreeSlotsPager
  112. lsp := dsm.logicalSlotsPager
  113. flsp := dsm.logicalFreeSlotsPager
  114. record, err := psp.StorageFile().Get(1)
  115. if err != nil {
  116. t.Error(err)
  117. return
  118. }
  119. _, err = dsm.Insert(&testutil.GobTestObject{Name: "test", EncErr: false, DecErr: false})
  120. if err != file.ErrAlreadyInUse {
  121. t.Error(err)
  122. return
  123. }
  124. psp.StorageFile().ReleaseInUse(record)
  125. record, err = lsp.StorageFile().Get(1)
  126. if err != nil {
  127. t.Error(err)
  128. return
  129. }
  130. _, err = dsm.Insert(&testutil.GobTestObject{Name: "test", EncErr: false, DecErr: false})
  131. if err != file.ErrAlreadyInUse {
  132. t.Error(err, loc)
  133. }
  134. lsp.StorageFile().ReleaseInUse(record)
  135. rpsp, _ := psp.StorageFile().Get(2)
  136. rfpsp, _ := fpsp.StorageFile().Get(1)
  137. rlsp, _ := lsp.StorageFile().Get(1)
  138. rflsp, _ := flsp.StorageFile().Get(1)
  139. err = dsm.Flush()
  140. if err.Error() != "Record is already in-use (storagemanagertest/test1.dbf - Record 1); "+
  141. "Record is already in-use (storagemanagertest/test1.ixf - Record 1); "+
  142. "Records are still in-use (storagemanagertest/test1.db - Records 1); "+
  143. "Records are still in-use (storagemanagertest/test1.dbf - Records 1); "+
  144. "Records are still in-use (storagemanagertest/test1.ix - Records 1); "+
  145. "Records are still in-use (storagemanagertest/test1.ixf - Records 1)" {
  146. t.Error(err)
  147. }
  148. err = dsm.Close()
  149. if err.Error() != "Records are still in-use (storagemanagertest/test1.db - Records 1); "+
  150. "Records are still in-use (storagemanagertest/test1.dbf - Records 1); "+
  151. "Records are still in-use (storagemanagertest/test1.ix - Records 1); "+
  152. "Records are still in-use (storagemanagertest/test1.ixf - Records 1)" {
  153. t.Error(err)
  154. }
  155. psp.StorageFile().ReleaseInUse(rpsp)
  156. fpsp.StorageFile().ReleaseInUse(rfpsp)
  157. lsp.StorageFile().ReleaseInUse(rlsp)
  158. flsp.StorageFile().ReleaseInUse(rflsp)
  159. _, err = dsm.FetchCached(0)
  160. if err != ErrNotInCache {
  161. t.Error("Unexpected FetchCached result:", err)
  162. return
  163. }
  164. if err = dsm.Close(); err != nil {
  165. t.Error(err)
  166. return
  167. }
  168. if !DataFileExist(DBDIR + "/test1") {
  169. t.Error("Main disk storage file was not detected.")
  170. return
  171. }
  172. if DataFileExist(DBDIR + "/" + InvalidFileName) {
  173. t.Error("Main disk storage file with invalid name should not exist.")
  174. return
  175. }
  176. }
  177. func testLockfileStartPanic(t *testing.T) {
  178. defer func() {
  179. if r := recover(); r == nil {
  180. t.Error("Attempting to open the same DiskStorageManager twice did not cause a panic.")
  181. }
  182. }()
  183. dsm := NewDiskStorageManager(DBDIR+"/test1", false, false, false, false)
  184. dsm.Close()
  185. }
  186. func TestDiskStorageManager2(t *testing.T) {
  187. var res string
  188. dsm := NewDiskStorageManager(DBDIR+"/test2", false, false, true, true)
  189. loc, err := dsm.Insert("This is a test")
  190. if err != nil {
  191. t.Error(err)
  192. return
  193. }
  194. checkLocation(t, loc, 1, 18)
  195. // Insert some filler data
  196. for i := 1; i < 253; i++ {
  197. loc2, err := dsm.Insert("1234")
  198. if err != nil {
  199. t.Error(err)
  200. return
  201. }
  202. checkLocation(t, loc2, 1, uint16(18+i*8))
  203. }
  204. record, _ := dsm.logicalSlotsSf.Get(2)
  205. _, err = dsm.Insert("This is a test")
  206. if err != file.ErrAlreadyInUse {
  207. t.Error(err)
  208. return
  209. }
  210. err = dsm.Fetch(util.PackLocation(2, 18), &res)
  211. if err != ErrSlotNotFound {
  212. t.Error(err)
  213. return
  214. }
  215. dsm.logicalSlotsSf.ReleaseInUse(record)
  216. err = dsm.Fetch(util.PackLocation(3, 18), &res)
  217. if err != ErrSlotNotFound {
  218. t.Error(err)
  219. return
  220. }
  221. record, _ = dsm.logicalSlotsSf.Get(1)
  222. err = dsm.Update(loc, "test")
  223. if err != file.ErrAlreadyInUse {
  224. t.Error(err)
  225. return
  226. }
  227. err = dsm.Fetch(loc, &res)
  228. if err != file.ErrAlreadyInUse {
  229. t.Error(err)
  230. return
  231. }
  232. dsm.logicalSlotsSf.ReleaseInUse(record)
  233. err = dsm.Update(util.PackLocation(2, 18), "test")
  234. if err != ErrSlotNotFound {
  235. t.Error(err)
  236. return
  237. }
  238. err = dsm.Update(loc, &testutil.GobTestObject{Name: "test", EncErr: true, DecErr: false})
  239. if err == nil {
  240. t.Error(err)
  241. return
  242. }
  243. err = dsm.Update(loc, &testutil.GobTestObject{Name: "test", EncErr: false, DecErr: false})
  244. if err != nil {
  245. t.Error(err)
  246. return
  247. }
  248. testres := &testutil.GobTestObject{Name: "test", EncErr: false, DecErr: true}
  249. err = dsm.Fetch(loc, &testres)
  250. if err == nil {
  251. t.Error("Unexpected decode result")
  252. return
  253. }
  254. record, _ = dsm.physicalSlotsSf.Get(1)
  255. var testres2 testutil.GobTestObject
  256. err = dsm.Fetch(loc, &testres2)
  257. if err != file.ErrAlreadyInUse {
  258. t.Error(err)
  259. return
  260. }
  261. // Test a normal update
  262. err = dsm.Update(loc, "tree")
  263. if err != file.ErrAlreadyInUse {
  264. t.Error(err)
  265. return
  266. }
  267. dsm.physicalSlotsSf.ReleaseInUse(record)
  268. err = dsm.Update(loc, "tree")
  269. if err != nil {
  270. t.Error(err)
  271. return
  272. }
  273. err = dsm.Fetch(loc, &res)
  274. if err != nil {
  275. t.Error(err)
  276. return
  277. }
  278. if res != "tree" {
  279. t.Error("Unexpected fetch result:", res)
  280. return
  281. }
  282. pl, _ := dsm.logicalSlotManager.Fetch(loc)
  283. if util.LocationRecord(pl) != 1 {
  284. t.Error("Unexpected initial location:", util.LocationRecord(pl))
  285. }
  286. _, err = dsm.Insert("test" + string(make([]byte, 10000)) + "test")
  287. if err != nil {
  288. t.Error(err)
  289. return
  290. }
  291. // Test reallocation
  292. err = dsm.Update(loc, "test"+string(make([]byte, 1000))+"test")
  293. if err != nil {
  294. t.Error(err)
  295. return
  296. }
  297. err = dsm.Update(loc, "test"+string(make([]byte, 1000))+"test")
  298. if err != nil {
  299. t.Error(err)
  300. return
  301. }
  302. pl, _ = dsm.logicalSlotManager.Fetch(loc)
  303. if util.LocationRecord(pl) != 2 {
  304. t.Error("Unexpected relocated location:", util.LocationRecord(pl))
  305. }
  306. if err = dsm.Flush(); err != nil {
  307. t.Error(err)
  308. return
  309. }
  310. if err = dsm.Close(); err != nil {
  311. t.Error(err)
  312. }
  313. // Reopen datastore readonly
  314. dsm = NewDiskStorageManager(DBDIR+"/test2", true, false, true, true)
  315. // Try write operations
  316. if l, err := dsm.Insert("Test"); l != 0 || err != ErrReadonly {
  317. t.Error("Unexpected result:", l, err)
  318. }
  319. if err := dsm.Update(loc, "Test"); err != ErrReadonly {
  320. t.Error("Unexpected result:", err)
  321. }
  322. if err := dsm.Free(loc); err != ErrReadonly {
  323. t.Error("Unexpected result:", err)
  324. }
  325. // NOP operations
  326. dsm.Rollback()
  327. dsm.SetRoot(1, 5)
  328. // Try reading
  329. if dsm.Root(1) != 1 {
  330. t.Error("Unexpected root:", dsm.Root(1))
  331. return
  332. }
  333. err = dsm.Fetch(util.PackLocation(1, 90), &res)
  334. if err != nil {
  335. t.Error(err)
  336. return
  337. }
  338. if res != "1234" {
  339. t.Error("Unexpected fetch result:", res)
  340. return
  341. }
  342. if dsm.Flush() != nil {
  343. t.Error("Flushing failed:", err)
  344. return
  345. }
  346. if err = dsm.Close(); err != nil {
  347. t.Error(err)
  348. }
  349. }
  350. func TestDiskStorageManager3(t *testing.T) {
  351. var res string
  352. dsm := NewDiskStorageManager(DBDIR+"/test3", false, false, true, true)
  353. if dsm.Free(util.PackLocation(2, 18)) != ErrSlotNotFound {
  354. t.Error("Unexpected free result")
  355. return
  356. }
  357. loc, err := dsm.Insert("This is a test")
  358. if err != nil {
  359. t.Error(err)
  360. return
  361. }
  362. dsm.Fetch(loc, &res)
  363. if res != "This is a test" {
  364. t.Error("Unexpected fetch result:", res)
  365. }
  366. record, _ := dsm.physicalSlotsSf.Get(1)
  367. if dsm.Free(loc) != file.ErrAlreadyInUse {
  368. t.Error("Unexpected free result")
  369. return
  370. }
  371. dsm.physicalSlotsSf.ReleaseInUse(record)
  372. record, _ = dsm.logicalSlotsSf.Get(1)
  373. if dsm.Free(loc) != file.ErrAlreadyInUse {
  374. t.Error("Unexpected free result")
  375. return
  376. }
  377. dsm.logicalSlotsSf.ReleaseInUse(record)
  378. if err := dsm.Free(loc); err != nil {
  379. t.Error(err)
  380. return
  381. }
  382. // Rollback call should have no effect
  383. if err := dsm.Rollback(); err != nil {
  384. t.Error(err)
  385. return
  386. }
  387. if err = dsm.Close(); err != nil {
  388. t.Error(err)
  389. return
  390. }
  391. }
  392. func TestDiskStorageManagerRollback(t *testing.T) {
  393. dsm := NewDiskStorageManager(DBDIR+"/test4", false, false, false, true)
  394. var res string
  395. // Test expected behaviour
  396. loc, err := dsm.Insert("This is a test")
  397. if err != nil {
  398. t.Error(err)
  399. }
  400. checkLocation(t, loc, 1, pageview.OffsetTransData)
  401. if err := dsm.Rollback(); err != nil {
  402. t.Error(err)
  403. return
  404. }
  405. dsm.Fetch(loc, &res)
  406. if res != "" {
  407. t.Error("Unexpected fetch result:", res)
  408. }
  409. loc, err = dsm.Insert("This is a test")
  410. if err != nil {
  411. t.Error(err)
  412. }
  413. checkLocation(t, loc, 1, pageview.OffsetTransData)
  414. if err := dsm.Flush(); err != nil {
  415. t.Error(err)
  416. return
  417. }
  418. if err := dsm.Rollback(); err != nil {
  419. t.Error(err)
  420. return
  421. }
  422. dsm.Fetch(loc, &res)
  423. if res != "This is a test" {
  424. t.Error("Unexpected fetch result:", res)
  425. }
  426. // Test error cases
  427. if err := dsm.Free(loc); err != nil {
  428. t.Error(err)
  429. return
  430. }
  431. psp := dsm.physicalSlotsPager
  432. fpsp := dsm.physicalFreeSlotsPager
  433. lsp := dsm.logicalSlotsPager
  434. flsp := dsm.logicalFreeSlotsPager
  435. rpsp, _ := psp.StorageFile().Get(1)
  436. rfpsp, _ := fpsp.StorageFile().Get(1)
  437. rlsp, _ := lsp.StorageFile().Get(1)
  438. rflsp, _ := flsp.StorageFile().Get(1)
  439. err = dsm.Rollback()
  440. if err.Error() != "Record is already in-use (storagemanagertest/test4.dbf - Record 1); "+
  441. "Record is already in-use (storagemanagertest/test4.ixf - Record 1); "+
  442. "Records are still in-use (storagemanagertest/test4.db - Records 1); "+
  443. "Records are still in-use (storagemanagertest/test4.dbf - Records 1); "+
  444. "Records are still in-use (storagemanagertest/test4.ix - Records 1); "+
  445. "Records are still in-use (storagemanagertest/test4.ixf - Records 1)" {
  446. t.Error(err)
  447. }
  448. psp.StorageFile().ReleaseInUse(rpsp)
  449. fpsp.StorageFile().ReleaseInUse(rfpsp)
  450. lsp.StorageFile().ReleaseInUse(rlsp)
  451. flsp.StorageFile().ReleaseInUse(rflsp)
  452. if err := dsm.Close(); err != nil {
  453. t.Error(err)
  454. return
  455. }
  456. }
  457. const InvalidFileName = "**" + string(0x0)
  458. func TestDiskStorageManagerInit(t *testing.T) {
  459. lockfile := lockutil.NewLockFile(DBDIR+"/"+"lock0.lck", time.Duration(50)*time.Millisecond)
  460. dsm := &DiskStorageManager{&ByteDiskStorageManager{DBDIR + "/" + InvalidFileName, false, true, true, &sync.Mutex{},
  461. nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, lockfile}}
  462. err := initByteDiskStorageManager(dsm.ByteDiskStorageManager)
  463. if err == nil {
  464. t.Error("Initialising a DiskStorageManager with an invalid filename should cause an error")
  465. return
  466. }
  467. testCannotInitPanic(t)
  468. dsm = &DiskStorageManager{&ByteDiskStorageManager{DBDIR + "/test999", false, true, true, &sync.Mutex{},
  469. nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil}}
  470. err = initByteDiskStorageManager(dsm.ByteDiskStorageManager)
  471. if err != nil {
  472. t.Error(err)
  473. return
  474. }
  475. dsm.SetRoot(RootIDVersion, VERSION+1)
  476. err = dsm.Close()
  477. if err != nil {
  478. t.Error(err)
  479. }
  480. testClosedPanic(t, dsm)
  481. testVersionCheckPanic(t)
  482. }
  483. func testCannotInitPanic(t *testing.T) {
  484. defer func() {
  485. if r := recover(); r == nil {
  486. t.Error("Creating DiskStorageManager with invalid filename did not cause a panic.")
  487. }
  488. }()
  489. NewDiskStorageManager(DBDIR+"/"+InvalidFileName, false, false, true, true)
  490. }
  491. func testClosedPanic(t *testing.T, dsm *DiskStorageManager) {
  492. defer func() {
  493. if r := recover(); r == nil {
  494. t.Error("Closing a closed DiskStorageManager did not cause a panic.")
  495. }
  496. }()
  497. dsm.Close()
  498. }
  499. func testVersionCheckPanic(t *testing.T) {
  500. dsm := &DiskStorageManager{&ByteDiskStorageManager{DBDIR + "/test999", false, true, true, &sync.Mutex{},
  501. nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil}}
  502. defer func() {
  503. if r := recover(); r == nil {
  504. t.Error("Wrong version did not cause a panic.")
  505. }
  506. }()
  507. initByteDiskStorageManager(dsm.ByteDiskStorageManager)
  508. }
  509. func checkLocation(t *testing.T, loc uint64, record uint64, offset uint16) {
  510. lrecord := util.LocationRecord(loc)
  511. loffset := util.LocationOffset(loc)
  512. if lrecord != record || loffset != offset {
  513. t.Error("Unexpected location. Expected:", record, offset, "Got:", lrecord, loffset)
  514. }
  515. }