transactionmanager_test.go 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610
  1. /*
  2. * EliasDB
  3. *
  4. * Copyright 2016 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the Mozilla Public
  7. * License, v. 2.0. If a copy of the MPL was not distributed with this
  8. * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  9. */
  10. package file
  11. import (
  12. "io"
  13. "os"
  14. "reflect"
  15. "testing"
  16. "devt.de/krotik/common/fileutil"
  17. "devt.de/krotik/common/testutil"
  18. )
  19. /*
  20. TestMain() which controls creation and deletion of DBDIR is defined in
  21. storagefile_test.go
  22. */
  23. func TestTransactionManagerInitialisation(t *testing.T) {
  24. if _, err := NewDefaultStorageFile(InvalidFileName, false); err == nil {
  25. t.Error("Invalid name for transaction log should cause an error")
  26. return
  27. }
  28. sf, err := NewDefaultStorageFile(DBDir+"/trans_test1", false)
  29. if err != nil {
  30. t.Error(err.Error())
  31. return
  32. }
  33. oldname := sf.name
  34. sf.name = InvalidFileName
  35. if _, err = NewTransactionManager(sf, true); err == nil {
  36. t.Error("Invalid name for transaction log should cause an error")
  37. return
  38. }
  39. if _, err = NewTransactionManager(sf, false); err == nil {
  40. t.Error("Invalid name for transaction log should cause an error")
  41. return
  42. }
  43. sf.name = oldname
  44. if sf.Name() != DBDir+"/trans_test1" {
  45. t.Error("Unexpected name of StorageFile:", sf.Name())
  46. return
  47. }
  48. if sf.RecordSize() != DefaultRecordSize {
  49. t.Error("Unexpected record size:", sf.RecordSize())
  50. return
  51. }
  52. tmName := sf.tm.name
  53. if err = sf.Close(); err != nil {
  54. t.Error(err)
  55. return
  56. }
  57. res, err := fileutil.PathExists(DBDir + "/trans_test1.0")
  58. if err != nil {
  59. t.Error(err)
  60. return
  61. }
  62. if !res {
  63. t.Error("Expected db file test1.0 does not exist")
  64. return
  65. }
  66. res, err = fileutil.PathExists(DBDir + "/trans_test1." + LogFileSuffix)
  67. if err != nil {
  68. t.Error(err)
  69. return
  70. }
  71. if !res {
  72. t.Error("Expected db file test1.0 does not exist")
  73. return
  74. }
  75. // Test Magic
  76. file, err := os.OpenFile(tmName, os.O_CREATE|os.O_TRUNC, 0660)
  77. if err != nil {
  78. t.Error(err)
  79. }
  80. file.Write([]byte{0x01, 0x02})
  81. file.Close()
  82. tm, err := NewTransactionManager(sf, true)
  83. if err != nil {
  84. t.Error(err)
  85. return
  86. }
  87. tm.close()
  88. file, err = os.OpenFile(tmName, os.O_RDONLY, 0660)
  89. if err != nil {
  90. t.Error(err)
  91. return
  92. }
  93. buf := make([]byte, 2)
  94. if _, err = file.Read(buf); err != nil {
  95. t.Error(err)
  96. return
  97. }
  98. if !reflect.DeepEqual(buf, TransactionLogHeader) {
  99. t.Error("Magic should have been restored in the transaction file")
  100. }
  101. if _, err = file.Read(buf); err != io.EOF {
  102. t.Error("File should only contain magic")
  103. }
  104. file.Close()
  105. // Next time we should still be able to open the file without problems
  106. tm, err = NewTransactionManager(sf, true)
  107. if err != nil {
  108. t.Error(err)
  109. return
  110. }
  111. tm.close()
  112. // Test corrupted transaction log
  113. file, err = os.OpenFile(tmName, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0660)
  114. if err != nil {
  115. t.Error(err)
  116. }
  117. file.Write(TransactionLogHeader)
  118. file.WriteString("*")
  119. file.Close()
  120. if _, err = NewTransactionManager(sf, true); err != io.ErrUnexpectedEOF {
  121. t.Error("Corrupted transaction logs should get an unexpected EOF", err)
  122. return
  123. }
  124. file, err = os.OpenFile(tmName, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0660)
  125. if err != nil {
  126. t.Error(err)
  127. }
  128. file.Write(TransactionLogHeader)
  129. file.Write([]byte{0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00})
  130. file.WriteString("HalloTEST")
  131. file.Close()
  132. if _, err = NewTransactionManager(sf, true); err != io.ErrUnexpectedEOF {
  133. t.Error("Corrupted transaction logs should get an unexpected EOF", err)
  134. return
  135. }
  136. }
  137. func TestTMSimpleHighLevelGetRelease(t *testing.T) {
  138. sf, err := NewDefaultStorageFile(DBDir+"/trans_test2", false)
  139. if err != nil {
  140. t.Error(err.Error())
  141. return
  142. }
  143. record, err := sf.Get(1)
  144. if err != nil {
  145. t.Error(err)
  146. return
  147. }
  148. record.WriteSingleByte(5, 0x42)
  149. sf.ReleaseInUse(record)
  150. if err = sf.Close(); err != nil {
  151. t.Error(err)
  152. return
  153. }
  154. // Check that all files are closed now on the sf
  155. l := len(sf.free) | len(sf.inUse) | len(sf.inTrans) | len(sf.dirty) | len(sf.files)
  156. if l != 0 {
  157. t.Error("Left over data in StorageFile:", sf)
  158. }
  159. // StorageFiles with transaction management cannot be reused after they
  160. // were closed.
  161. sf, err = NewDefaultStorageFile(DBDir+"/trans_test2", false)
  162. if err != nil {
  163. t.Error(err.Error())
  164. return
  165. }
  166. record, err = sf.Get(1)
  167. if err != nil {
  168. t.Error(err)
  169. return
  170. }
  171. if record.ReadSingleByte(5) != 0x42 {
  172. t.Error("Unexpected value in record")
  173. }
  174. sf.ReleaseInUse(record)
  175. sf.Close()
  176. l = len(sf.free) | len(sf.inUse) | len(sf.inTrans) |
  177. len(sf.dirty) | len(sf.files)
  178. if l != 0 {
  179. t.Error("Left over data in StorageFile:", sf)
  180. }
  181. }
  182. func TestTMComplexHighLevelGetRelease(t *testing.T) {
  183. // Test the auto commit of many transactions
  184. sf, err := NewDefaultStorageFile(DBDir+"/trans_test3", false)
  185. if err != nil {
  186. t.Error(err.Error())
  187. return
  188. }
  189. // Releasing a nil pointer should have no effect
  190. sf.releaseInTrans(nil, true)
  191. for i := 0; i < DefaultTransInLog+1; i++ {
  192. record, err := sf.Get(1 + uint64(i))
  193. if err != nil {
  194. t.Error(err)
  195. return
  196. }
  197. record.WriteSingleByte(5+i, 0x42)
  198. sf.ReleaseInUse(record)
  199. if i == DefaultTransInLog {
  200. if len(sf.inTrans) != DefaultTransInLog {
  201. t.Error("Expected", DefaultTransInLog, "records in transaction")
  202. }
  203. }
  204. sf.Flush()
  205. if i == DefaultTransInLog {
  206. if len(sf.inTrans) != 1 && len(sf.free) == 10 {
  207. t.Error("Expected", DefaultTransInLog, "records to be free")
  208. }
  209. out := sf.String()
  210. if out != "Storage File: storagefiletest/trans_test3 (transDisabled:false recordSize:4096 maxFileSize:9999998976)\n"+
  211. "====\n"+
  212. "Free Records: 1, 2, 3, 4, 5, 6, 7, 8, 9, 10\n"+
  213. "InUse Records: \n"+
  214. "InTrans Records: 11\n"+
  215. "Dirty Records: \n"+
  216. "Open files: storagefiletest/trans_test3.0 (0)\n"+
  217. "====\n"+
  218. "Transaction Manager: storagefiletest/trans_test3.tlg (logFile:true curTrans:0 maxTrans:10)\n"+
  219. "====\n"+
  220. "transList:\n"+
  221. "0: 11 \n"+
  222. "1: \n"+
  223. "2: \n"+
  224. "3: \n"+
  225. "4: \n"+
  226. "5: \n"+
  227. "6: \n"+
  228. "7: \n"+
  229. "8: \n"+
  230. "9: \n"+
  231. "====\n" {
  232. t.Error("Unexpected output of storage file:", out)
  233. return
  234. }
  235. // Do one more and check that a free record is being reused
  236. record, err := sf.Get(1 + uint64(i+1))
  237. if err != nil {
  238. t.Error(err)
  239. return
  240. }
  241. if len(sf.free) != DefaultTransInLog-1 {
  242. t.Error("Expected that a free record would be reused")
  243. }
  244. for _, b := range record.data {
  245. if b != 0x00 {
  246. t.Error("Reused record was not cleaned properly:", record)
  247. return
  248. }
  249. }
  250. record.WriteSingleByte(5+i+1, 0x42)
  251. sf.ReleaseInUse(record)
  252. sf.Flush()
  253. }
  254. }
  255. sf.Close()
  256. tm, err := NewTransactionManager(sf, true)
  257. if err != nil {
  258. t.Error(err)
  259. return
  260. }
  261. defer tm.logFile.Close()
  262. sf.tm = tm
  263. sf.name = InvalidFileName
  264. record := NewRecord(5, make([]byte, sf.recordSize, sf.recordSize))
  265. tm.transList[0] = append(tm.transList[0], record)
  266. tm.transList[1] = append(tm.transList[1], record)
  267. record.transCount = 2
  268. if err := tm.syncLogFromMemory(); err == nil {
  269. t.Error("Writing records to an invalid storage file should fail")
  270. }
  271. if record.transCount != 1 {
  272. t.Error("Transaction count in record should have been decreased")
  273. }
  274. // tm.logFile is now nil since there was an error
  275. tm.logFile = testutil.NewTestingFile(10)
  276. tm.curTrans = 5
  277. tm.start()
  278. tm.add(record)
  279. if err := tm.commit(); err == nil {
  280. t.Error("Failed write operations should be reported")
  281. }
  282. tm.logFile = testutil.NewTestingFile(3)
  283. if err := tm.commit(); err == nil {
  284. t.Error("Failed write operations should be reported")
  285. }
  286. }
  287. func TestRecover(t *testing.T) {
  288. sf, err := NewDefaultStorageFile(DBDir+"/trans_test4", false)
  289. if err != nil {
  290. t.Error(err.Error())
  291. return
  292. }
  293. record, err := sf.Get(1)
  294. if err != nil {
  295. t.Error(err)
  296. return
  297. }
  298. record.WriteSingleByte(5, 0x42)
  299. sf.ReleaseInUse(record)
  300. sf.Flush()
  301. // Getting the record for a read operation and then releasing it
  302. // should have no effect on its membership in the transaction
  303. record, err = sf.Get(1)
  304. if err != nil {
  305. t.Error(err)
  306. return
  307. }
  308. sf.ReleaseInUse(record)
  309. if sf.inTrans[record.ID()] == nil {
  310. t.Error("Record should still be part of the transaction")
  311. return
  312. }
  313. record, err = sf.Get(2)
  314. if err != nil {
  315. t.Error(err)
  316. return
  317. }
  318. record.WriteSingleByte(6, 0x42)
  319. sf.ReleaseInUse(record)
  320. // Not let an error happen which makes the transaction log file unavailable
  321. sf.tm.logFile.Close()
  322. if sf.Flush() == nil {
  323. t.Error("Flush should fail when the transaction log cannot be accessed")
  324. }
  325. record, err = sf.Get(2)
  326. if err != nil {
  327. t.Error(err)
  328. return
  329. }
  330. record.WriteSingleByte(7, 0x42)
  331. sf.ReleaseInUse(record)
  332. // The record should still be in a transaction and should have now both changes
  333. if record.ReadSingleByte(5) != 0 || record.ReadSingleByte(6) != 0x42 ||
  334. record.ReadSingleByte(7) != 0x42 {
  335. t.Error("Unexpected data in record:", record)
  336. return
  337. }
  338. if err = sf.Close(); err == nil {
  339. t.Error(err)
  340. return
  341. }
  342. // Now lets get out of the mess
  343. sf.transDisabled = true
  344. if err = sf.Close(); err != ErrInTrans {
  345. t.Error(err)
  346. return
  347. }
  348. sf.inTrans = make(map[uint64]*Record)
  349. if err = sf.Close(); err != nil {
  350. t.Error(err)
  351. return
  352. }
  353. // Check that all files are closed now on the sf
  354. l := len(sf.free) | len(sf.inUse) | len(sf.inTrans) | len(sf.dirty) | len(sf.files)
  355. if l != 0 {
  356. t.Error("Left over data in StorageFile:", sf)
  357. }
  358. // Open the StorageFile again and hope that recover() does the right thing
  359. sf, err = NewDefaultStorageFile(DBDir+"/trans_test4", false)
  360. if err != nil {
  361. t.Error(err.Error())
  362. return
  363. }
  364. record, err = sf.Get(1)
  365. if err != nil {
  366. t.Error(err)
  367. return
  368. }
  369. record2, err := sf.Get(2)
  370. if err != nil {
  371. t.Error(err)
  372. return
  373. }
  374. // Check that expected values are there / not there
  375. if record.ReadSingleByte(5) != 0x42 || record.ReadSingleByte(6) != 0 ||
  376. record.ReadSingleByte(7) != 0 {
  377. t.Error("Unexpected data in record1:", record)
  378. return
  379. }
  380. // All transactions on record2 should have failed
  381. if record2.ReadSingleByte(5) != 0 || record2.ReadSingleByte(6) != 0 ||
  382. record2.ReadSingleByte(7) != 0 {
  383. t.Error("Unexpected data in record2:", record)
  384. return
  385. }
  386. sf.ReleaseInUse(record)
  387. sf.ReleaseInUse(record2)
  388. sf.Close()
  389. // Check that all files are closed now on the sf
  390. l = len(sf.free) | len(sf.inUse) | len(sf.inTrans) | len(sf.dirty) | len(sf.files)
  391. if l != 0 {
  392. t.Error("Left over data in StorageFile:", sf)
  393. }
  394. }
  395. func TestRollback(t *testing.T) {
  396. sf, err := NewDefaultStorageFile(DBDir+"/trans_test5", false)
  397. if err != nil {
  398. t.Error(err.Error())
  399. return
  400. }
  401. record, err := sf.Get(1)
  402. if err != nil {
  403. t.Error(err)
  404. return
  405. }
  406. record.WriteSingleByte(5, 0x42)
  407. sf.ReleaseInUse(record)
  408. sf.Flush()
  409. record, err = sf.Get(2)
  410. if err != nil {
  411. t.Error(err)
  412. return
  413. }
  414. record.WriteSingleByte(6, 0x42)
  415. sf.ReleaseInUse(record)
  416. if err := sf.Rollback(); err != nil {
  417. t.Error(err)
  418. return
  419. }
  420. if err = sf.Close(); err != nil {
  421. t.Error(err)
  422. return
  423. }
  424. sf, err = NewDefaultStorageFile(DBDir+"/trans_test2", false)
  425. if err != nil {
  426. t.Error(err.Error())
  427. return
  428. }
  429. record, err = sf.Get(1)
  430. if err != nil {
  431. t.Error(err)
  432. return
  433. }
  434. record2, err := sf.Get(2)
  435. if err != nil {
  436. t.Error(err)
  437. return
  438. }
  439. // Check that expected values are there / not there
  440. if record.ReadSingleByte(5) != 0x42 || record.ReadSingleByte(6) != 0 {
  441. t.Error("Unexpected data in record1:", record)
  442. return
  443. }
  444. // All transactions on record2 should have failed
  445. if record2.ReadSingleByte(5) != 0 || record2.ReadSingleByte(6) != 0 {
  446. t.Error("Unexpected data in record2:", record)
  447. return
  448. }
  449. sf.ReleaseInUse(record)
  450. sf.ReleaseInUse(record2)
  451. sf.Close()
  452. }
  453. func TestRollbackFail(t *testing.T) {
  454. sf, err := NewDefaultStorageFile(DBDir+"/trans_test6", false)
  455. if err != nil {
  456. t.Error(err.Error())
  457. return
  458. }
  459. record, err := sf.Get(1)
  460. if err != nil {
  461. t.Error(err)
  462. return
  463. }
  464. record.WriteSingleByte(5, 0x42)
  465. if err = sf.Rollback(); err != ErrInUse {
  466. t.Error("It should not be possible to rollback while records are still in use")
  467. }
  468. sf.ReleaseInUse(record)
  469. sf.tm.logFile.Close()
  470. sf.tm.name = DBDir + "/" + InvalidFileName + "." + LogFileSuffix
  471. if err = sf.Rollback(); err == nil {
  472. t.Error("Rollback should fail when using invalid filename for transaction log")
  473. return
  474. }
  475. // Cleanup
  476. sf.transDisabled = true
  477. sf.Close()
  478. sf, err = NewDefaultStorageFile(DBDir+"/trans_test6", false)
  479. if err != nil {
  480. t.Error(err.Error())
  481. return
  482. }
  483. sf.inTrans[record.ID()] = record
  484. if err = sf.Rollback(); err != ErrInTrans {
  485. t.Error("It should not be possible to rollback while records are still in transaction")
  486. return
  487. }
  488. delete(sf.inTrans, record.ID())
  489. sf.Close()
  490. }