storagefile_test.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667
  1. /*
  2. * EliasDB
  3. *
  4. * Copyright 2016 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the Mozilla Public
  7. * License, v. 2.0. If a copy of the MPL was not distributed with this
  8. * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  9. */
  10. package file
  11. import (
  12. "flag"
  13. "fmt"
  14. "os"
  15. "testing"
  16. "devt.de/krotik/common/fileutil"
  17. )
  18. const DBDir = "storagefiletest"
  19. const InvalidFileName = "**" + string(0x0)
  20. func TestMain(m *testing.M) {
  21. flag.Parse()
  22. // Setup
  23. if res, _ := fileutil.PathExists(DBDir); res {
  24. os.RemoveAll(DBDir)
  25. }
  26. err := os.Mkdir(DBDir, 0770)
  27. if err != nil {
  28. fmt.Print("Could not create test directory:", err.Error())
  29. os.Exit(1)
  30. }
  31. // Run the tests
  32. res := m.Run()
  33. // Teardown
  34. err = os.RemoveAll(DBDir)
  35. if err != nil {
  36. fmt.Print("Could not remove test directory:", err.Error())
  37. }
  38. os.Exit(res)
  39. }
  40. func TestStorageFileInitialisation(t *testing.T) {
  41. // \0 and / are the only illegal characters for filenames in unix
  42. sf, err := NewDefaultStorageFile(DBDir+"/"+InvalidFileName, true)
  43. if err == nil {
  44. t.Error("Invalid name should cause an error")
  45. return
  46. }
  47. sf, err = NewDefaultStorageFile(DBDir+"/test1", true)
  48. if err != nil {
  49. t.Error(err.Error())
  50. return
  51. }
  52. if sf.Name() != DBDir+"/test1" {
  53. t.Error("Unexpected name of StorageFile:", sf.Name())
  54. return
  55. }
  56. if sf.RecordSize() != DefaultRecordSize {
  57. t.Error("Unexpected record size:", sf.RecordSize())
  58. return
  59. }
  60. defer sf.Close()
  61. res, err := fileutil.PathExists(DBDir + "/test1.0")
  62. if err != nil {
  63. t.Error(err)
  64. return
  65. }
  66. if !res {
  67. t.Error("Expected db file test1.0 does not exist")
  68. return
  69. }
  70. if len(sf.files) != 1 {
  71. t.Error("Unexpected number of files in StorageFile:", sf.files)
  72. return
  73. }
  74. }
  75. func TestGetFile(t *testing.T) {
  76. sf := &StorageFile{DBDir + "/test2", true, 10, 10, nil, nil, nil, nil,
  77. make([]*os.File, 0), nil}
  78. defer sf.Close()
  79. file, err := sf.getFile(0)
  80. if err != nil {
  81. t.Error(err.Error())
  82. return
  83. }
  84. if file.Name() != DBDir+"/test2.0" {
  85. t.Error("Unexpected file from getFile")
  86. return
  87. }
  88. checkFilesArray(t, sf, 1, 0, DBDir+"/test2.0")
  89. file, err = sf.getFile(42)
  90. if err != nil {
  91. t.Error(err.Error())
  92. return
  93. }
  94. if file.Name() != DBDir+"/test2.4" {
  95. t.Error("Unexpected file from getFile")
  96. return
  97. }
  98. checkFilesArray(t, sf, 5, 0, DBDir+"/test2.0")
  99. checkFilesArray(t, sf, 5, 1, "")
  100. checkFilesArray(t, sf, 5, 2, "")
  101. checkFilesArray(t, sf, 5, 3, "")
  102. checkFilesArray(t, sf, 5, 4, DBDir+"/test2.4")
  103. file, err = sf.getFile(25)
  104. if err != nil {
  105. t.Error(err.Error())
  106. return
  107. }
  108. if file.Name() != DBDir+"/test2.2" {
  109. t.Error("Unexpected file from getFile")
  110. return
  111. }
  112. checkFilesArray(t, sf, 5, 0, DBDir+"/test2.0")
  113. checkFilesArray(t, sf, 5, 1, "")
  114. checkFilesArray(t, sf, 5, 2, DBDir+"/test2.2")
  115. checkFilesArray(t, sf, 5, 3, "")
  116. checkFilesArray(t, sf, 5, 4, DBDir+"/test2.4")
  117. file, err = sf.getFile(11)
  118. if err != nil {
  119. t.Error(err.Error())
  120. return
  121. }
  122. if file.Name() != DBDir+"/test2.1" {
  123. t.Error("Unexpected file from getFile")
  124. return
  125. }
  126. checkFilesArray(t, sf, 5, 0, DBDir+"/test2.0")
  127. checkFilesArray(t, sf, 5, 1, DBDir+"/test2.1")
  128. checkFilesArray(t, sf, 5, 2, DBDir+"/test2.2")
  129. checkFilesArray(t, sf, 5, 3, "")
  130. checkFilesArray(t, sf, 5, 4, DBDir+"/test2.4")
  131. file, err = sf.getFile(49)
  132. if err != nil {
  133. t.Error(err.Error())
  134. return
  135. }
  136. if file.Name() != DBDir+"/test2.4" {
  137. t.Error("Unexpected file from getFile")
  138. return
  139. }
  140. checkFilesArray(t, sf, 5, 0, DBDir+"/test2.0")
  141. checkFilesArray(t, sf, 5, 1, DBDir+"/test2.1")
  142. checkFilesArray(t, sf, 5, 2, DBDir+"/test2.2")
  143. checkFilesArray(t, sf, 5, 3, "")
  144. checkFilesArray(t, sf, 5, 4, DBDir+"/test2.4")
  145. }
  146. func checkFilesArray(t *testing.T, sf *StorageFile, explen int, pos int, name string) {
  147. if len(sf.files) != explen {
  148. t.Error("Unexpected files array:", sf.files, " expected size:", explen)
  149. }
  150. file := sf.files[pos]
  151. if name == "" && file != nil {
  152. t.Error("Unexpected file at pos:", pos, " name:", file.Name())
  153. } else if name != "" && file == nil {
  154. t.Error("Unexpected nil pointer at pos:", pos, " expected name:", name)
  155. } else if file != nil && name != file.Name() {
  156. t.Error("Unexpected file at pos:", pos, " name:", file.Name(), " expected name:", name)
  157. }
  158. }
  159. func TestLowLevelReadWrite(t *testing.T) {
  160. // Create a new record and write it
  161. sf, err := NewDefaultStorageFile(DBDir+"/test3", true)
  162. if err != nil {
  163. t.Error(err.Error())
  164. return
  165. }
  166. record := sf.createRecord(1)
  167. record.WriteSingleByte(5, 0x42)
  168. oldfiles := sf.files
  169. sf.name = DBDir + "/" + InvalidFileName
  170. sf.files = make([]*os.File, 0)
  171. _, err = sf.Get(1)
  172. if err == nil {
  173. t.Error("Invalid filename should cause an error")
  174. return
  175. }
  176. err = sf.writeRecord(record)
  177. if err == nil {
  178. t.Error("Invalid filename should cause an error")
  179. return
  180. }
  181. sf.name = DBDir + "/test3"
  182. sf.files = oldfiles
  183. err = sf.writeRecord(record)
  184. if err != nil {
  185. t.Error("Writing with a correct name should succeed", err)
  186. return
  187. }
  188. sf.Close()
  189. sf, err = NewDefaultStorageFile(DBDir+"/test3", true)
  190. if err != nil {
  191. t.Error(err.Error())
  192. return
  193. }
  194. record = sf.createRecord(1)
  195. oldfiles = sf.files
  196. sf.name = DBDir + "/" + InvalidFileName
  197. sf.files = make([]*os.File, 0)
  198. record.data = nil
  199. err = sf.readRecord(record)
  200. if err != ErrNilData {
  201. t.Error("Nil pointer in record data should cause an error")
  202. return
  203. }
  204. record.ClearData()
  205. err = sf.readRecord(record)
  206. if err == nil {
  207. t.Error("Invalid filename should cause an error")
  208. return
  209. }
  210. sf.name = DBDir + "/test3"
  211. sf.files = oldfiles
  212. oldrecordSize := sf.recordSize
  213. sf.recordSize = DefaultRecordSize - 1
  214. testReadRecordPanic(t, sf, record)
  215. sf.recordSize = oldrecordSize
  216. err = sf.readRecord(record)
  217. if err != nil {
  218. t.Error("Reading with a correct name should succeed")
  219. return
  220. }
  221. sf.Close()
  222. if record.ReadSingleByte(5) != 0x42 {
  223. t.Error("Couldn't read byte which was written before.")
  224. return
  225. }
  226. }
  227. func testReadRecordPanic(t *testing.T, sf *StorageFile, r *Record) {
  228. defer func() {
  229. if r := recover(); r == nil {
  230. t.Error("Changing of the record size did not cause a panic.")
  231. }
  232. }()
  233. sf.readRecord(r)
  234. }
  235. func TestHighLevelGetRelease(t *testing.T) {
  236. // Create some records and write to them
  237. sf, err := NewDefaultStorageFile(DBDir+"/test4", true)
  238. if err != nil {
  239. t.Error(err.Error())
  240. return
  241. }
  242. // Get records and check that the expected files are there
  243. record1, err := sf.Get(1)
  244. if err != nil {
  245. t.Error(err)
  246. return
  247. }
  248. checkPath(t, "test4.0")
  249. checkMap(t, &sf.inUse, record1.ID(), true, "Record1", "in use")
  250. record2, err := sf.Get((DefaultFileSize/DefaultRecordSize)*4 + 5)
  251. if err != nil {
  252. t.Error(err)
  253. return
  254. }
  255. checkPath(t, "test4.4")
  256. checkMap(t, &sf.inUse, record2.ID(), true, "Record2", "in use")
  257. record3, err := sf.Get(2)
  258. if err != nil {
  259. t.Error(err)
  260. return
  261. }
  262. checkPath(t, "test4.0")
  263. // Make sure the retrieved records are marked in use
  264. checkMap(t, &sf.inUse, record3.ID(), true, "Record3", "in use")
  265. checkMap(t, &sf.free, record1.ID(), false, "Record1", "in use")
  266. checkMap(t, &sf.free, record2.ID(), false, "Record2", "in use")
  267. checkMap(t, &sf.free, record3.ID(), false, "Record3", "in use")
  268. // Now use the records and release them
  269. record1.WriteUInt16(2, 0x4268)
  270. record1.WriteUInt16(10, 0x66)
  271. record2.WriteInt32(11, -0x7654321)
  272. sf.ReleaseInUse(record2)
  273. // A rollback should have no consequences
  274. sf.Rollback()
  275. // Check that the records have been released and scheduled for write
  276. // (i.e. they are in the dirty table)
  277. checkMap(t, &sf.dirty, record2.ID(), true, "Record2", "dirty")
  278. checkMap(t, &sf.inUse, record2.ID(), false, "Record2", "in use")
  279. _, err = sf.Get((DefaultFileSize/DefaultRecordSize)*4 + 5)
  280. if err != nil {
  281. t.Error(err)
  282. return
  283. }
  284. checkMap(t, &sf.dirty, record2.ID(), false, "Record2", "dirty")
  285. checkMap(t, &sf.inUse, record2.ID(), true, "Record2", "in use")
  286. sf.ReleaseInUse(record1)
  287. checkMap(t, &sf.dirty, record1.ID(), true, "Record1", "dirty")
  288. checkMap(t, &sf.inUse, record1.ID(), false, "Record1", "in use")
  289. checkMap(t, &sf.dirty, record2.ID(), false, "Record2", "dirty")
  290. checkMap(t, &sf.inUse, record2.ID(), true, "Record2", "in use")
  291. checkMap(t, &sf.dirty, record3.ID(), false, "Record3", "dirty")
  292. checkMap(t, &sf.inUse, record3.ID(), true, "Record3", "in use")
  293. sf.ReleaseInUseID(record2.ID(), true)
  294. checkMap(t, &sf.dirty, record1.ID(), true, "Record1", "dirty")
  295. checkMap(t, &sf.inUse, record1.ID(), false, "Record1", "in use")
  296. checkMap(t, &sf.dirty, record2.ID(), true, "Record2", "dirty")
  297. checkMap(t, &sf.inUse, record2.ID(), false, "Record2", "in use")
  298. checkMap(t, &sf.dirty, record3.ID(), false, "Record3", "dirty")
  299. checkMap(t, &sf.inUse, record3.ID(), true, "Record3", "in use")
  300. if err = sf.Flush(); err != ErrInUse {
  301. t.Error("StorageFile should complain about records being in use")
  302. }
  303. record4, err := sf.Get(5)
  304. if err != nil {
  305. t.Error(err)
  306. return
  307. }
  308. sf.ReleaseInUse(record3)
  309. checkMap(t, &sf.dirty, record1.ID(), true, "Record1", "dirty")
  310. checkMap(t, &sf.inUse, record1.ID(), false, "Record1", "in use")
  311. checkMap(t, &sf.dirty, record2.ID(), true, "Record2", "dirty")
  312. checkMap(t, &sf.inUse, record2.ID(), false, "Record2", "in use")
  313. // Check that a record which has not been written to is put into the
  314. // free map
  315. checkMap(t, &sf.free, record3.ID(), true, "Record3", "free")
  316. checkMap(t, &sf.dirty, record3.ID(), false, "Record3", "dirty")
  317. checkMap(t, &sf.inUse, record3.ID(), false, "Record3", "in use")
  318. // Test string representation of the StorageFile
  319. if sf.String() != "Storage File: storagefiletest/test4 "+
  320. "(transDisabled:true recordSize:4096 maxFileSize:9999998976)\n"+
  321. "====\n"+
  322. "Free Records: 2\n"+
  323. "InUse Records: 5\n"+
  324. "InTrans Records: \n"+
  325. "Dirty Records: 1, 9765629\n"+
  326. "Open files: storagefiletest/test4.0 (0), storagefiletest/test4.4 (4)\n"+
  327. "====\n" {
  328. t.Error("Unexpected string representation of StorageFile:", sf.String())
  329. }
  330. sf.ReleaseInUse(record4)
  331. // Check that after the changes have been written to disk that
  332. // all records are in the free map
  333. sf.Flush()
  334. checkMap(t, &sf.dirty, record1.ID(), false, "Record1", "dirty")
  335. checkMap(t, &sf.inUse, record1.ID(), false, "Record1", "in use")
  336. checkMap(t, &sf.free, record1.ID(), true, "Record1", "in use")
  337. checkMap(t, &sf.dirty, record2.ID(), false, "Record2", "dirty")
  338. checkMap(t, &sf.inUse, record2.ID(), false, "Record2", "in use")
  339. checkMap(t, &sf.free, record2.ID(), true, "Record2", "in use")
  340. checkMap(t, &sf.dirty, record3.ID(), false, "Record3", "dirty")
  341. checkMap(t, &sf.inUse, record3.ID(), false, "Record3", "in use")
  342. checkMap(t, &sf.free, record3.ID(), true, "Record3", "in use")
  343. _, err = sf.Get((DefaultFileSize/DefaultRecordSize)*4 + 5)
  344. if err != nil {
  345. t.Error(err)
  346. return
  347. }
  348. checkMap(t, &sf.free, record2.ID(), false, "Record2", "free")
  349. checkMap(t, &sf.inUse, record2.ID(), true, "Record2", "in use")
  350. sf.ReleaseInUse(record2)
  351. checkMap(t, &sf.free, record2.ID(), true, "Record2", "free")
  352. checkMap(t, &sf.inUse, record2.ID(), false, "Record2", "in use")
  353. sf.Close()
  354. // Open the storage file again with a different object and
  355. // try to read back the written
  356. sf, err = NewDefaultStorageFile(DBDir+"/test4", true)
  357. if err != nil {
  358. t.Error(err.Error())
  359. return
  360. }
  361. record1, err = sf.Get(1)
  362. if err != nil {
  363. t.Error(err)
  364. return
  365. }
  366. // Test that requesting a record twice without releasing it causes an error.
  367. if _, err = sf.Get(1); err != ErrAlreadyInUse {
  368. t.Error("Requesting a record which is already in use should cause an error")
  369. }
  370. if err.Error() != "Record is already in-use (storagefiletest/test4 - Record 1)" {
  371. t.Error("Unexpected error string:", err)
  372. return
  373. }
  374. checkMap(t, &sf.inUse, record1.ID(), true, "Record1", "in use")
  375. record2, err = sf.Get((DefaultFileSize/DefaultRecordSize)*4 + 5)
  376. if err != nil {
  377. t.Error(err)
  378. return
  379. }
  380. checkMap(t, &sf.inUse, record2.ID(), true, "Record2", "in use")
  381. // Check that we can read back the written data
  382. if d := record1.ReadUInt16(2); d != 0x4268 {
  383. t.Error("Expected value in record1 not found")
  384. return
  385. }
  386. if d := record1.ReadUInt16(10); d != 0x66 {
  387. t.Error("Expected value in record1 not found")
  388. return
  389. }
  390. if d := record2.ReadInt32(11); d != -0x7654321 {
  391. t.Error("Expected value in record1 not found", d)
  392. return
  393. }
  394. sf.ReleaseInUse(record1)
  395. // Since record3 was just created and is empty it should not be in use
  396. record3 = sf.createRecord(5)
  397. checkMap(t, &sf.inUse, record3.ID(), false, "Record3", "in use")
  398. // Record2 has been used
  399. checkMap(t, &sf.inUse, record2.ID(), true, "Record2", "in use")
  400. // An attempt to close the file should return an error
  401. err = sf.Close()
  402. if err != ErrInUse {
  403. t.Error("Attempting to close a StorageFile with records in use should " +
  404. "return an error")
  405. return
  406. }
  407. sf.ReleaseInUse(record2)
  408. err = sf.Close()
  409. if err != nil {
  410. t.Error(err)
  411. return
  412. }
  413. }
  414. func checkPath(t *testing.T, path string) {
  415. res, err := fileutil.PathExists(DBDir + "/" + path)
  416. if err != nil {
  417. t.Error(err)
  418. }
  419. if !res {
  420. t.Error("Expected db file", path, "does not exist")
  421. }
  422. }
  423. func checkMap(t *testing.T, mapvar *map[uint64]*Record, id uint64, expected bool,
  424. name string, mapname string) {
  425. if _, ok := (*mapvar)[id]; expected != ok {
  426. if expected {
  427. t.Error(name, "should be", mapname)
  428. } else {
  429. t.Error(name, "should not be", mapname)
  430. }
  431. }
  432. }
  433. func TestFlushingClosing(t *testing.T) {
  434. sf, err := NewDefaultStorageFile(DBDir+"/test5", true)
  435. if err != nil {
  436. t.Error(err.Error())
  437. return
  438. }
  439. if sf.Flush() != nil {
  440. t.Error("Flushing an unused file should not cause an error")
  441. return
  442. }
  443. record, err := sf.Get(1)
  444. if err != nil {
  445. t.Error(err)
  446. return
  447. }
  448. record.WriteSingleByte(0, 0)
  449. if sf.Flush() != ErrInUse {
  450. t.Error("Flushing should not be allowed while records are in use")
  451. return
  452. }
  453. sf.ReleaseInUse(nil) // This should not cause a panic
  454. if sf.ReleaseInUseID(5000, true) != ErrNotInUse {
  455. t.Error("It should not be possible to release records which are not in use")
  456. return
  457. }
  458. record.ClearDirty()
  459. sf.ReleaseInUseID(1, true)
  460. if !record.Dirty() {
  461. t.Error("Record should be marked as dirty after it was released as dirty")
  462. return
  463. }
  464. testReleasePanic(t, sf, record)
  465. // Modifying the id at this point causes unspecified behaviour. Once a
  466. // record was released it should not be modified.
  467. record.data = nil
  468. err = sf.Flush()
  469. if err != ErrNilData {
  470. t.Error("It should not be possible to flush a record with an invalid id to disk")
  471. return
  472. }
  473. checkMap(t, &sf.dirty, 1, true, "Record1", "dirty")
  474. // Get the record again and discard it
  475. record, err = sf.Get(1)
  476. if err != nil {
  477. t.Error(err)
  478. return
  479. }
  480. checkMap(t, &sf.inUse, 1, true, "Record1", "in use")
  481. // Need to correct the id otherwise the discarding will not work
  482. record.SetID(1)
  483. sf.Discard(nil) // This should not cause a panic
  484. sf.Discard(record)
  485. checkMap(t, &sf.dirty, 1, false, "Record1", "dirty")
  486. checkMap(t, &sf.inUse, 1, false, "Record1", "in use")
  487. sf.Sync() // This should just complete and not cause a panic
  488. record, err = sf.Get(5)
  489. if err != nil {
  490. t.Error(err)
  491. return
  492. }
  493. // This should be possible even if the record is not dirty at all
  494. sf.ReleaseInUseID(record.ID(), true)
  495. checkMap(t, &sf.dirty, 5, true, "Record1", "dirty")
  496. recordData := record.data
  497. record.data = nil
  498. if sf.Close() != ErrNilData {
  499. t.Error("Closing with a dirty record with negative id should not be possible", err)
  500. return
  501. }
  502. record.data = recordData
  503. err = sf.Close()
  504. if err != nil {
  505. t.Error(err)
  506. return
  507. }
  508. // Make sure the close call did flush dirty records
  509. checkMap(t, &sf.dirty, 5, false, "Record1", "dirty")
  510. }
  511. func testReleasePanic(t *testing.T, sf *StorageFile, r *Record) {
  512. defer func() {
  513. if r := recover(); r == nil {
  514. t.Error("Releaseing a record multiple times without using it " +
  515. "did not cause a panic.")
  516. }
  517. }()
  518. sf.ReleaseInUse(r)
  519. }