multifilebuffer.go 10.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441
  1. /*
  2. * Public Domain Software
  3. *
  4. * I (Matthias Ladkau) am the author of the source code in this file.
  5. * I have placed the source code in this file in the public domain.
  6. *
  7. * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
  8. */
  9. package fileutil
  10. import (
  11. "fmt"
  12. "io/ioutil"
  13. "os"
  14. "path"
  15. "sort"
  16. "strconv"
  17. "strings"
  18. "sync"
  19. "devt.de/krotik/common/timeutil"
  20. )
  21. /*
  22. MultiFileBuffer is a file-persitent buffer which can be split over multiple files.
  23. A specified file is opened and used as backend storage for a byte buffer. By
  24. default, the file grows indefinitely. It is possible to specify a rollover
  25. condition to allow the file to rollover once the condition is satisfied.
  26. If the condition is satisfied, the file is closed and a new file is silently
  27. opened for output. The buffer will save old log files by appending the
  28. extensions ‘.1’, ‘.2’ etc., to the file name. The rollover condition is only
  29. checked once at the beginning of a write operation.
  30. For example, with a base file name of app.log, the buffer would create
  31. app.log, app.log.1, app.log.2, etc. The file being written to is always app.log.
  32. When this file is filled, it is closed and renamed to app.log.1, and if files
  33. app.log.1, app.log.2, etc. exist, then they are renamed to app.log.2, app.log.3
  34. etc. respectively.
  35. */
  36. type MultiFileBuffer struct {
  37. lock *sync.Mutex // Lock for reading and writing
  38. filename string // File name for buffer
  39. basename string // Base file name (file name + iterator decoration)
  40. iterator FilenameIterator // Iterator for file names
  41. cond RolloverCondition // Rollover condition
  42. fp *os.File // Current file handle
  43. }
  44. /*
  45. NewMultiFileBuffer creates a new MultiFileBuffer with a given file name
  46. iterator and rollover condition.
  47. */
  48. func NewMultiFileBuffer(filename string, it FilenameIterator, cond RolloverCondition) (*MultiFileBuffer, error) {
  49. var err error
  50. mfb := &MultiFileBuffer{&sync.Mutex{}, filename, it.Basename(filename), it, cond, nil}
  51. if err = mfb.checkrollover(); err != nil {
  52. return nil, err
  53. }
  54. if mfb.fp == nil {
  55. // File existed and can be continued
  56. mfb.lock.Lock()
  57. mfb.fp, err = os.OpenFile(mfb.basename, os.O_APPEND|os.O_RDWR, 0660)
  58. mfb.lock.Unlock()
  59. }
  60. return mfb, err
  61. }
  62. /*
  63. Write writes len(p) bytes from p to the underlying data stream. It returns
  64. the number of bytes written from p (0 <= n <= len(p)) and any error
  65. encountered that caused the write to stop early.
  66. */
  67. func (mfb *MultiFileBuffer) Write(output []byte) (int, error) {
  68. var b int
  69. err := mfb.checkrollover()
  70. if err == nil {
  71. if mfb.fp == nil {
  72. // File existed and can be continued
  73. mfb.lock.Lock()
  74. mfb.fp, err = os.OpenFile(mfb.basename, os.O_APPEND|os.O_RDWR, 0660)
  75. mfb.lock.Unlock()
  76. }
  77. if err == nil {
  78. mfb.lock.Lock()
  79. b, err = mfb.fp.Write(output)
  80. mfb.lock.Unlock()
  81. }
  82. }
  83. return b, err
  84. }
  85. /*
  86. checkrollover checks if the buffer files should be switched.
  87. */
  88. func (mfb *MultiFileBuffer) checkrollover() error {
  89. mfb.lock.Lock()
  90. defer mfb.lock.Unlock()
  91. // Update basename here
  92. mfb.basename = mfb.iterator.Basename(mfb.filename)
  93. // Rollover if the base file does not exist
  94. ex, err := PathExists(mfb.basename)
  95. if err == nil && (!ex || mfb.cond.CheckRollover(mfb.basename)) {
  96. // Rollover if either the base file does not exist or the
  97. // rollover condition is satisfied
  98. err = mfb.rollover()
  99. }
  100. return err
  101. }
  102. /*
  103. Close closes the buffer.
  104. */
  105. func (mfb *MultiFileBuffer) Close() error {
  106. var err error
  107. if mfb.fp != nil {
  108. err = mfb.fp.Close()
  109. mfb.fp = nil
  110. }
  111. return err
  112. }
  113. /*
  114. rollover switches the buffer files.
  115. */
  116. func (mfb *MultiFileBuffer) rollover() error {
  117. var err error
  118. // Recursive file renaming function
  119. var ensureFileSlot func(fn string) error
  120. ensureFileSlot = func(fn string) error {
  121. // Check if the file exists already
  122. ex, err := PathExists(fn)
  123. if ex && err == nil {
  124. // Determine new file name
  125. newfn := mfb.iterator.NextName(fn)
  126. if newfn == "" {
  127. // If it is the end of the iteration just delete the file
  128. err = os.Remove(fn)
  129. } else {
  130. // Ensure the new file name is usable
  131. err = ensureFileSlot(newfn)
  132. // Rename file according to iterator.NextName()
  133. if err == nil {
  134. err = os.Rename(fn, newfn)
  135. }
  136. }
  137. }
  138. return err
  139. }
  140. // Close existing file
  141. err = mfb.Close()
  142. // Create file handle
  143. if err == nil {
  144. err = ensureFileSlot(mfb.basename)
  145. if err == nil {
  146. // Overwrite existing base file
  147. mfb.fp, err = os.OpenFile(mfb.basename, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0660)
  148. }
  149. }
  150. return err
  151. }
  152. // Rollover conditions
  153. // ===================
  154. /*
  155. RolloverCondition is used by the MultiFileBuffer to check if the buffer files
  156. should be switched.
  157. */
  158. type RolloverCondition interface {
  159. /*
  160. CheckRollover checks if the buffer files should be switched.
  161. */
  162. CheckRollover(basename string) bool
  163. }
  164. /*
  165. EmptyRolloverCondition creates a rollover condition which is never true.
  166. */
  167. func EmptyRolloverCondition() RolloverCondition {
  168. return &emptyRolloverCondition{}
  169. }
  170. /*
  171. emptyRolloverCondition is a rollover condition which is never true.
  172. */
  173. type emptyRolloverCondition struct {
  174. }
  175. /*
  176. NextName returns the next file name based on the current file name.
  177. An empty string means the end of the iteration.
  178. */
  179. func (rc *emptyRolloverCondition) CheckRollover(basename string) bool {
  180. return false
  181. }
  182. /*
  183. SizeBasedRolloverCondition creates a new rollover condition based on file
  184. size. The condition is satisfied if the base file exceeds a certain file size.
  185. */
  186. func SizeBasedRolloverCondition(maxSize int64) RolloverCondition {
  187. return &sizeBasedRolloverCondition{maxSize}
  188. }
  189. /*
  190. sizeBasedRolloverCondition is the implementation of the size based rollover
  191. condition.
  192. */
  193. type sizeBasedRolloverCondition struct {
  194. maxSize int64
  195. }
  196. /*
  197. NextName returns the next file name based on the current file name.
  198. An empty string means the end of the iteration.
  199. */
  200. func (rc *sizeBasedRolloverCondition) CheckRollover(basename string) bool {
  201. ret := false
  202. if info, err := os.Stat(basename); err == nil {
  203. ret = info.Size() >= rc.maxSize
  204. }
  205. return ret
  206. }
  207. // FilenameIterator
  208. // ================
  209. /*
  210. FilenameIterator is used by the MultiFileBuffer to determine the new file name
  211. when rotating the buffer files. Basename is called before doing any calculation.
  212. This function should do general filename decoration. If the decoration changes
  213. over time then the function needs to also handle the cleanup.
  214. */
  215. type FilenameIterator interface {
  216. /*
  217. Basename decorades the initial file name.
  218. */
  219. Basename(filename string) string
  220. /*
  221. NextName returns the next file name based on the current file name.
  222. An empty string means the end of the iteration.
  223. */
  224. NextName(currentName string) string
  225. }
  226. /*
  227. ConsecutiveNumberIterator creates a new file name iterator which adds numbers
  228. at the end of files. Up to maxNum files will be created. A maxNum parameter
  229. < 1 means there is no limit.
  230. */
  231. func ConsecutiveNumberIterator(maxNum int) FilenameIterator {
  232. return &consecutiveNumberIterator{maxNum}
  233. }
  234. /*
  235. consecutiveNumberIterator is the implementation of the consecutive number
  236. file iterator.
  237. */
  238. type consecutiveNumberIterator struct {
  239. maxNum int
  240. }
  241. /*
  242. Basename decorades the initial file name.
  243. */
  244. func (it *consecutiveNumberIterator) Basename(filename string) string {
  245. return filename
  246. }
  247. /*
  248. NextName returns the next file name based on the current file name.
  249. An empty string means the end of the iteration.
  250. */
  251. func (it *consecutiveNumberIterator) NextName(currentName string) string {
  252. if i := strings.LastIndex(currentName, "."); i > 0 {
  253. if num, err := strconv.ParseInt(currentName[i+1:], 10, 64); err == nil {
  254. nextNum := int(num + 1)
  255. if it.maxNum > 0 && nextNum > it.maxNum {
  256. return ""
  257. }
  258. return fmt.Sprintf("%s.%v", currentName[:i], nextNum)
  259. }
  260. }
  261. return fmt.Sprintf("%s.1", currentName)
  262. }
  263. /*
  264. DailyDateIterator creates a new file name iterator which adds dates at the
  265. end of files. The log will be switched at least once every day. Up to maxNumPerDay
  266. files will be created per day. A maxNumPerDay parameter < 1 means there is no limit.
  267. Up to maxDays different days will be kept (oldest ones are deleted). A maxDays
  268. parameter < 1 means everything is kept.
  269. */
  270. func DailyDateIterator(maxNumPerDay int, maxDays int) FilenameIterator {
  271. return &dailyDateIterator{&consecutiveNumberIterator{maxNumPerDay}, maxDays, timeutil.MakeTimestamp}
  272. }
  273. /*
  274. consecutiveNumberIterator is the implementation of the consecutive number
  275. file iterator.
  276. */
  277. type dailyDateIterator struct {
  278. *consecutiveNumberIterator
  279. maxDays int
  280. tsFunc func() string // Timestamp function
  281. }
  282. /*
  283. NextName returns the next file name based on the current file name.
  284. An empty string means the end of the iteration.
  285. */
  286. func (it *dailyDateIterator) Basename(filename string) string {
  287. // Get todays date
  288. ts := it.tsFunc()
  289. today, _ := timeutil.TimestampString(ts, "UTC")
  290. today = today[:10]
  291. // Cleanup old files
  292. if it.maxDays > 0 {
  293. prefix := path.Base(filename)
  294. dir := path.Dir(filename)
  295. if files, err := ioutil.ReadDir(dir); err == nil {
  296. var datesToConsider []string
  297. // Collect all relevant files
  298. foundToday := false
  299. for _, f := range files {
  300. if strings.HasPrefix(f.Name(), prefix) && len(f.Name()) > len(prefix) {
  301. dateString := f.Name()[len(prefix)+1:]
  302. if !strings.ContainsRune(dateString, '.') {
  303. datesToConsider = append(datesToConsider, dateString)
  304. if !foundToday {
  305. foundToday = dateString == today
  306. }
  307. }
  308. }
  309. }
  310. // Make sure today is one of the dates
  311. if !foundToday {
  312. datesToConsider = append(datesToConsider, today)
  313. }
  314. // Sort them so the newest ones are kept
  315. sort.Strings(datesToConsider)
  316. // Check if files need to be removed
  317. if len(datesToConsider) > it.maxDays {
  318. datesToRemove := datesToConsider[:len(datesToConsider)-it.maxDays]
  319. for _, f := range files {
  320. for _, dateToRemove := range datesToRemove {
  321. if strings.HasPrefix(f.Name(), fmt.Sprintf("%s.%s", prefix, dateToRemove)) {
  322. os.Remove(path.Join(dir, f.Name()))
  323. }
  324. }
  325. }
  326. }
  327. }
  328. }
  329. return fmt.Sprintf("%s.%s", filename, today)
  330. }