diskstoragemanager_test.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668
  1. /*
  2. * EliasDB
  3. *
  4. * Copyright 2016 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the Mozilla Public
  7. * License, v. 2.0. If a copy of the MPL was not distributed with this
  8. * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  9. */
  10. package storage
  11. import (
  12. "bytes"
  13. "encoding/gob"
  14. "os"
  15. "sync"
  16. "testing"
  17. "time"
  18. "devt.de/krotik/common/lockutil"
  19. "devt.de/krotik/common/testutil"
  20. "devt.de/krotik/eliasdb/storage/file"
  21. "devt.de/krotik/eliasdb/storage/slotting/pageview"
  22. "devt.de/krotik/eliasdb/storage/util"
  23. )
  24. func TestDiskStorageManagerLockFilePanic(t *testing.T) {
  25. dsm := NewDiskStorageManager(DBDIR+"/test0", false, false, true, false)
  26. defer func() {
  27. if r := recover(); r == nil {
  28. t.Error("Doing an operation with a storagefile with stopped lock watcher goroutine did not cause a panic.")
  29. }
  30. // Start the watcher goroutine again and close the file
  31. dsm.lockfile.Start()
  32. dsm.Close()
  33. }()
  34. file, err := os.OpenFile(DBDIR+"/test0.lck", os.O_CREATE|os.O_TRUNC, 0660)
  35. if err != nil {
  36. t.Error(err)
  37. return
  38. }
  39. file.WriteString("t")
  40. file.Close()
  41. // Give the watcher goroutine time to die
  42. time.Sleep(time.Duration(100) * time.Millisecond)
  43. if dsm.lockfile.WatcherRunning() {
  44. t.Error("Watcher goroutine did not die")
  45. }
  46. // This should cause a panic
  47. dsm.Free(util.PackLocation(2, 20))
  48. }
  49. func TestDiskStorageManager1(t *testing.T) {
  50. var res string
  51. dsm := NewDiskStorageManager(DBDIR+"/test1", false, false, true, false)
  52. if dsm.Name() != "DiskStorageFile:"+DBDIR+"/test1" {
  53. t.Error("Unexpected name for DiskStorageManager:", dsm.Name())
  54. return
  55. }
  56. // Make sure that another process which attempts to open the same
  57. // storage would panic
  58. time.Sleep(100 * time.Millisecond) // Need some time here otherwise Windows fails sometimes
  59. testLockfileStartPanic(t)
  60. // Test simple insert
  61. loc, err := dsm.Insert("This is a test")
  62. if err != nil {
  63. t.Error(err)
  64. }
  65. checkLocation(t, loc, 1, pageview.OffsetTransData)
  66. dsm.Fetch(loc, &res)
  67. if res != "This is a test" {
  68. t.Error("Unexpected fetch result:", res)
  69. }
  70. // Test case where we give a byte slice
  71. var res2 string
  72. bs := make([]byte, 18)
  73. dsm.ByteDiskStorageManager.Fetch(loc, bs)
  74. err = gob.NewDecoder(bytes.NewReader(bs)).Decode(&res2)
  75. if err != nil || string(res2) != "This is a test" {
  76. t.Error("Unexpected fetch result:", err, res2)
  77. }
  78. // Get physical slot for stored data
  79. ploc, err := dsm.logicalSlotManager.Fetch(loc)
  80. if err != nil {
  81. t.Error(err)
  82. return
  83. }
  84. // Test uodate
  85. // The next update should allocate a new physical record
  86. err = dsm.Update(loc, "This is another test")
  87. if err != nil {
  88. t.Error(err)
  89. }
  90. // Get new physical slot for stored data
  91. newPloc, err := dsm.logicalSlotManager.Fetch(loc)
  92. if err != nil {
  93. t.Error(err)
  94. return
  95. }
  96. if ploc == newPloc {
  97. t.Error("Physical address should have changed")
  98. return
  99. }
  100. dsm.Fetch(loc, &res)
  101. if res != "This is another test" {
  102. t.Error("Unexpected fetch result:", res)
  103. }
  104. // Test insert error
  105. _, err = dsm.Insert(&testutil.GobTestObject{Name: "test", EncErr: true, DecErr: false})
  106. if err == nil {
  107. t.Error(err)
  108. return
  109. }
  110. psp := dsm.physicalSlotsPager
  111. fpsp := dsm.physicalFreeSlotsPager
  112. lsp := dsm.logicalSlotsPager
  113. flsp := dsm.logicalFreeSlotsPager
  114. record, err := psp.StorageFile().Get(1)
  115. if err != nil {
  116. t.Error(err)
  117. return
  118. }
  119. _, err = dsm.Insert(&testutil.GobTestObject{Name: "test", EncErr: false, DecErr: false})
  120. if sfe, ok := err.(*file.StorageFileError); !ok || sfe.Type != file.ErrAlreadyInUse {
  121. t.Error(err)
  122. return
  123. }
  124. psp.StorageFile().ReleaseInUse(record)
  125. record, err = lsp.StorageFile().Get(1)
  126. if err != nil {
  127. t.Error(err)
  128. return
  129. }
  130. _, err = dsm.Insert(&testutil.GobTestObject{Name: "test", EncErr: false, DecErr: false})
  131. if sfe, ok := err.(*file.StorageFileError); !ok || sfe.Type != file.ErrAlreadyInUse {
  132. t.Error(err, loc)
  133. }
  134. lsp.StorageFile().ReleaseInUse(record)
  135. rpsp, _ := psp.StorageFile().Get(2)
  136. rfpsp, _ := fpsp.StorageFile().Get(1)
  137. rlsp, _ := lsp.StorageFile().Get(1)
  138. rflsp, _ := flsp.StorageFile().Get(1)
  139. err = dsm.Flush()
  140. if err.Error() != "Record is already in-use (storagemanagertest/test1.dbf - Record 1); "+
  141. "Record is already in-use (storagemanagertest/test1.ixf - Record 1); "+
  142. "Records are still in-use (storagemanagertest/test1.db - Records 1); "+
  143. "Records are still in-use (storagemanagertest/test1.dbf - Records 1); "+
  144. "Records are still in-use (storagemanagertest/test1.ix - Records 1); "+
  145. "Records are still in-use (storagemanagertest/test1.ixf - Records 1)" {
  146. t.Error(err)
  147. }
  148. err = dsm.Close()
  149. if err.Error() != "Records are still in-use (storagemanagertest/test1.db - Records 1); "+
  150. "Records are still in-use (storagemanagertest/test1.dbf - Records 1); "+
  151. "Records are still in-use (storagemanagertest/test1.ix - Records 1); "+
  152. "Records are still in-use (storagemanagertest/test1.ixf - Records 1)" {
  153. t.Error(err)
  154. }
  155. psp.StorageFile().ReleaseInUse(rpsp)
  156. fpsp.StorageFile().ReleaseInUse(rfpsp)
  157. lsp.StorageFile().ReleaseInUse(rlsp)
  158. flsp.StorageFile().ReleaseInUse(rflsp)
  159. _, err = dsm.FetchCached(0)
  160. if err != ErrNotInCache {
  161. t.Error("Unexpected FetchCached result:", err)
  162. return
  163. }
  164. if err = dsm.Close(); err != nil {
  165. t.Error(err)
  166. return
  167. }
  168. if !DataFileExist(DBDIR + "/test1") {
  169. t.Error("Main disk storage file was not detected.")
  170. return
  171. }
  172. if DataFileExist(DBDIR + "/" + InvalidFileName) {
  173. t.Error("Main disk storage file with invalid name should not exist.")
  174. return
  175. }
  176. }
  177. func testLockfileStartPanic(t *testing.T) {
  178. defer func() {
  179. if r := recover(); r == nil {
  180. t.Error("Attempting to open the same DiskStorageManager twice did not cause a panic.")
  181. }
  182. }()
  183. dsm := NewDiskStorageManager(DBDIR+"/test1", false, false, false, false)
  184. dsm.Close()
  185. }
  186. func TestDiskStorageManager2(t *testing.T) {
  187. var res string
  188. dsm := NewDiskStorageManager(DBDIR+"/test2", false, false, true, true)
  189. loc, err := dsm.Insert("This is a test")
  190. if err != nil {
  191. t.Error(err)
  192. return
  193. }
  194. checkLocation(t, loc, 1, 18)
  195. // Insert some filler data
  196. for i := 1; i < 253; i++ {
  197. loc2, err := dsm.Insert("1234")
  198. if err != nil {
  199. t.Error(err)
  200. return
  201. }
  202. checkLocation(t, loc2, 1, uint16(18+i*8))
  203. }
  204. record, _ := dsm.logicalSlotsSf.Get(2)
  205. _, err = dsm.Insert("This is a test")
  206. if sfe, ok := err.(*file.StorageFileError); !ok || sfe.Type != file.ErrAlreadyInUse {
  207. t.Error(err)
  208. return
  209. }
  210. err = dsm.Fetch(util.PackLocation(2, 18), &res)
  211. if err != ErrSlotNotFound {
  212. t.Error(err)
  213. return
  214. }
  215. dsm.logicalSlotsSf.ReleaseInUse(record)
  216. err = dsm.Fetch(util.PackLocation(3, 18), &res)
  217. if err != ErrSlotNotFound {
  218. t.Error(err)
  219. return
  220. }
  221. record, _ = dsm.logicalSlotsSf.Get(1)
  222. err = dsm.Update(loc, "test")
  223. if sfe, ok := err.(*file.StorageFileError); !ok || sfe.Type != file.ErrAlreadyInUse {
  224. t.Error(err)
  225. return
  226. }
  227. err = dsm.Fetch(loc, &res)
  228. if sfe, ok := err.(*file.StorageFileError); !ok || sfe.Type != file.ErrAlreadyInUse {
  229. t.Error(err)
  230. return
  231. }
  232. dsm.logicalSlotsSf.ReleaseInUse(record)
  233. err = dsm.Update(util.PackLocation(2, 18), "test")
  234. if err != ErrSlotNotFound {
  235. t.Error(err)
  236. return
  237. }
  238. err = dsm.Update(loc, &testutil.GobTestObject{Name: "test", EncErr: true, DecErr: false})
  239. if err == nil {
  240. t.Error(err)
  241. return
  242. }
  243. err = dsm.Update(loc, &testutil.GobTestObject{Name: "test", EncErr: false, DecErr: false})
  244. if err != nil {
  245. t.Error(err)
  246. return
  247. }
  248. testres := &testutil.GobTestObject{Name: "test", EncErr: false, DecErr: true}
  249. err = dsm.Fetch(loc, &testres)
  250. if err == nil {
  251. t.Error("Unexpected decode result")
  252. return
  253. }
  254. record, _ = dsm.physicalSlotsSf.Get(1)
  255. var testres2 testutil.GobTestObject
  256. err = dsm.Fetch(loc, &testres2)
  257. if sfe, ok := err.(*file.StorageFileError); !ok || sfe.Type != file.ErrAlreadyInUse {
  258. t.Error(err)
  259. return
  260. }
  261. // Test a normal update
  262. err = dsm.Update(loc, "tree")
  263. if sfe, ok := err.(*file.StorageFileError); !ok || sfe.Type != file.ErrAlreadyInUse {
  264. t.Error(err)
  265. return
  266. }
  267. dsm.physicalSlotsSf.ReleaseInUse(record)
  268. err = dsm.Update(loc, "tree")
  269. if err != nil {
  270. t.Error(err)
  271. return
  272. }
  273. err = dsm.Fetch(loc, &res)
  274. if err != nil {
  275. t.Error(err)
  276. return
  277. }
  278. if res != "tree" {
  279. t.Error("Unexpected fetch result:", res)
  280. return
  281. }
  282. pl, _ := dsm.logicalSlotManager.Fetch(loc)
  283. if util.LocationRecord(pl) != 1 {
  284. t.Error("Unexpected initial location:", util.LocationRecord(pl))
  285. }
  286. _, err = dsm.Insert("test" + string(make([]byte, 10000)) + "test")
  287. if err != nil {
  288. t.Error(err)
  289. return
  290. }
  291. // Test reallocation
  292. err = dsm.Update(loc, "test"+string(make([]byte, 1000))+"test")
  293. if err != nil {
  294. t.Error(err)
  295. return
  296. }
  297. err = dsm.Update(loc, "test"+string(make([]byte, 1000))+"test")
  298. if err != nil {
  299. t.Error(err)
  300. return
  301. }
  302. pl, _ = dsm.logicalSlotManager.Fetch(loc)
  303. if util.LocationRecord(pl) != 2 {
  304. t.Error("Unexpected relocated location:", util.LocationRecord(pl))
  305. }
  306. if err = dsm.Flush(); err != nil {
  307. t.Error(err)
  308. return
  309. }
  310. if err = dsm.Close(); err != nil {
  311. t.Error(err)
  312. }
  313. // Reopen datastore readonly
  314. dsm = NewDiskStorageManager(DBDIR+"/test2", true, false, true, true)
  315. // Try write operations
  316. if l, err := dsm.Insert("Test"); l != 0 || err != ErrReadonly {
  317. t.Error("Unexpected result:", l, err)
  318. }
  319. if err := dsm.Update(loc, "Test"); err != ErrReadonly {
  320. t.Error("Unexpected result:", err)
  321. }
  322. if err := dsm.Free(loc); err != ErrReadonly {
  323. t.Error("Unexpected result:", err)
  324. }
  325. // NOP operations
  326. dsm.Rollback()
  327. dsm.SetRoot(1, 5)
  328. // Try reading
  329. if dsm.Root(1) != 1 {
  330. t.Error("Unexpected root:", dsm.Root(1))
  331. return
  332. }
  333. err = dsm.Fetch(util.PackLocation(1, 90), &res)
  334. if err != nil {
  335. t.Error(err)
  336. return
  337. }
  338. if res != "1234" {
  339. t.Error("Unexpected fetch result:", res)
  340. return
  341. }
  342. if dsm.Flush() != nil {
  343. t.Error("Flushing failed:", err)
  344. return
  345. }
  346. if err = dsm.Close(); err != nil {
  347. t.Error(err)
  348. }
  349. }
  350. func TestDiskStorageManager3(t *testing.T) {
  351. var res string
  352. dsm := NewDiskStorageManager(DBDIR+"/test3", false, false, true, true)
  353. if dsm.Free(util.PackLocation(2, 18)) != ErrSlotNotFound {
  354. t.Error("Unexpected free result")
  355. return
  356. }
  357. loc, err := dsm.Insert("This is a test")
  358. if err != nil {
  359. t.Error(err)
  360. return
  361. }
  362. dsm.Fetch(loc, &res)
  363. if res != "This is a test" {
  364. t.Error("Unexpected fetch result:", res)
  365. }
  366. record, _ := dsm.physicalSlotsSf.Get(1)
  367. err = dsm.Free(loc)
  368. if sfe, ok := err.(*file.StorageFileError); !ok || sfe.Type != file.ErrAlreadyInUse {
  369. t.Error("Unexpected free result")
  370. return
  371. }
  372. dsm.physicalSlotsSf.ReleaseInUse(record)
  373. record, _ = dsm.logicalSlotsSf.Get(1)
  374. err = dsm.Free(loc)
  375. if sfe, ok := err.(*file.StorageFileError); !ok || sfe.Type != file.ErrAlreadyInUse {
  376. t.Error("Unexpected free result")
  377. return
  378. }
  379. dsm.logicalSlotsSf.ReleaseInUse(record)
  380. if err := dsm.Free(loc); err != nil {
  381. t.Error(err)
  382. return
  383. }
  384. // Rollback call should have no effect
  385. if err := dsm.Rollback(); err != nil {
  386. t.Error(err)
  387. return
  388. }
  389. if err = dsm.Close(); err != nil {
  390. t.Error(err)
  391. return
  392. }
  393. }
  394. func TestDiskStorageManagerRollback(t *testing.T) {
  395. dsm := NewDiskStorageManager(DBDIR+"/test4", false, false, false, true)
  396. var res string
  397. // Test expected behaviour
  398. loc, err := dsm.Insert("This is a test")
  399. if err != nil {
  400. t.Error(err)
  401. }
  402. checkLocation(t, loc, 1, pageview.OffsetTransData)
  403. if err := dsm.Rollback(); err != nil {
  404. t.Error(err)
  405. return
  406. }
  407. dsm.Fetch(loc, &res)
  408. if res != "" {
  409. t.Error("Unexpected fetch result:", res)
  410. }
  411. loc, err = dsm.Insert("This is a test")
  412. if err != nil {
  413. t.Error(err)
  414. }
  415. checkLocation(t, loc, 1, pageview.OffsetTransData)
  416. if err := dsm.Flush(); err != nil {
  417. t.Error(err)
  418. return
  419. }
  420. if err := dsm.Rollback(); err != nil {
  421. t.Error(err)
  422. return
  423. }
  424. dsm.Fetch(loc, &res)
  425. if res != "This is a test" {
  426. t.Error("Unexpected fetch result:", res)
  427. }
  428. // Test error cases
  429. if err := dsm.Free(loc); err != nil {
  430. t.Error(err)
  431. return
  432. }
  433. psp := dsm.physicalSlotsPager
  434. fpsp := dsm.physicalFreeSlotsPager
  435. lsp := dsm.logicalSlotsPager
  436. flsp := dsm.logicalFreeSlotsPager
  437. rpsp, _ := psp.StorageFile().Get(1)
  438. rfpsp, _ := fpsp.StorageFile().Get(1)
  439. rlsp, _ := lsp.StorageFile().Get(1)
  440. rflsp, _ := flsp.StorageFile().Get(1)
  441. err = dsm.Rollback()
  442. if err.Error() != "Record is already in-use (storagemanagertest/test4.dbf - Record 1); "+
  443. "Record is already in-use (storagemanagertest/test4.ixf - Record 1); "+
  444. "Records are still in-use (storagemanagertest/test4.db - Records 1); "+
  445. "Records are still in-use (storagemanagertest/test4.dbf - Records 1); "+
  446. "Records are still in-use (storagemanagertest/test4.ix - Records 1); "+
  447. "Records are still in-use (storagemanagertest/test4.ixf - Records 1)" {
  448. t.Error(err)
  449. }
  450. psp.StorageFile().ReleaseInUse(rpsp)
  451. fpsp.StorageFile().ReleaseInUse(rfpsp)
  452. lsp.StorageFile().ReleaseInUse(rlsp)
  453. flsp.StorageFile().ReleaseInUse(rflsp)
  454. if err := dsm.Close(); err != nil {
  455. t.Error(err)
  456. return
  457. }
  458. }
  459. const InvalidFileName = "**" + "\x00"
  460. func TestDiskStorageManagerInit(t *testing.T) {
  461. lockfile := lockutil.NewLockFile(DBDIR+"/"+"lock0.lck", time.Duration(50)*time.Millisecond)
  462. dsm := &DiskStorageManager{&ByteDiskStorageManager{DBDIR + "/" + InvalidFileName, false, true, true, &sync.Mutex{},
  463. nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, lockfile}}
  464. err := initByteDiskStorageManager(dsm.ByteDiskStorageManager)
  465. if err == nil {
  466. t.Error("Initialising a DiskStorageManager with an invalid filename should cause an error")
  467. return
  468. }
  469. testCannotInitPanic(t)
  470. dsm = &DiskStorageManager{&ByteDiskStorageManager{DBDIR + "/test999", false, true, true, &sync.Mutex{},
  471. nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil}}
  472. err = initByteDiskStorageManager(dsm.ByteDiskStorageManager)
  473. if err != nil {
  474. t.Error(err)
  475. return
  476. }
  477. dsm.SetRoot(RootIDVersion, VERSION+1)
  478. err = dsm.Close()
  479. if err != nil {
  480. t.Error(err)
  481. }
  482. testClosedPanic(t, dsm)
  483. testVersionCheckPanic(t)
  484. }
  485. func testCannotInitPanic(t *testing.T) {
  486. defer func() {
  487. if r := recover(); r == nil {
  488. t.Error("Creating DiskStorageManager with invalid filename did not cause a panic.")
  489. }
  490. }()
  491. NewDiskStorageManager(DBDIR+"/"+InvalidFileName, false, false, true, true)
  492. }
  493. func testClosedPanic(t *testing.T, dsm *DiskStorageManager) {
  494. defer func() {
  495. if r := recover(); r == nil {
  496. t.Error("Closing a closed DiskStorageManager did not cause a panic.")
  497. }
  498. }()
  499. dsm.Close()
  500. }
  501. func testVersionCheckPanic(t *testing.T) {
  502. dsm := &DiskStorageManager{&ByteDiskStorageManager{DBDIR + "/test999", false, true, true, &sync.Mutex{},
  503. nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil}}
  504. defer func() {
  505. if r := recover(); r == nil {
  506. t.Error("Wrong version did not cause a panic.")
  507. }
  508. }()
  509. initByteDiskStorageManager(dsm.ByteDiskStorageManager)
  510. }
  511. func checkLocation(t *testing.T, loc uint64, record uint64, offset uint16) {
  512. lrecord := util.LocationRecord(loc)
  513. loffset := util.LocationOffset(loc)
  514. if lrecord != record || loffset != offset {
  515. t.Error("Unexpected location. Expected:", record, offset, "Got:", lrecord, loffset)
  516. }
  517. }