/* * Public Domain Software * * I (Matthias Ladkau) am the author of the source code in this file. * I have placed the source code in this file in the public domain. * * For further information see: http://creativecommons.org/publicdomain/zero/1.0/ */ package fileutil import ( "fmt" "io/ioutil" "os" "path" "sort" "strconv" "strings" "sync" "devt.de/krotik/common/timeutil" ) /* MultiFileBuffer is a file-persitent buffer which can be split over multiple files. A specified file is opened and used as backend storage for a byte buffer. By default, the file grows indefinitely. It is possible to specify a rollover condition to allow the file to rollover once the condition is satisfied. If the condition is satisfied, the file is closed and a new file is silently opened for output. The buffer will save old log files by appending the extensions ‘.1’, ‘.2’ etc., to the file name. The rollover condition is only checked once at the beginning of a write operation. For example, with a base file name of app.log, the buffer would create app.log, app.log.1, app.log.2, etc. The file being written to is always app.log. When this file is filled, it is closed and renamed to app.log.1, and if files app.log.1, app.log.2, etc. exist, then they are renamed to app.log.2, app.log.3 etc. respectively. */ type MultiFileBuffer struct { lock *sync.Mutex // Lock for reading and writing filename string // File name for buffer basename string // Base file name (file name + iterator decoration) iterator FilenameIterator // Iterator for file names cond RolloverCondition // Rollover condition fp *os.File // Current file handle } /* NewMultiFileBuffer creates a new MultiFileBuffer with a given file name iterator and rollover condition. */ func NewMultiFileBuffer(filename string, it FilenameIterator, cond RolloverCondition) (*MultiFileBuffer, error) { var err error mfb := &MultiFileBuffer{&sync.Mutex{}, filename, it.Basename(filename), it, cond, nil} if err = mfb.checkrollover(); err != nil { return nil, err } if mfb.fp == nil { // File existed and can be continued mfb.lock.Lock() mfb.fp, err = os.OpenFile(mfb.basename, os.O_APPEND|os.O_RDWR, 0660) mfb.lock.Unlock() } return mfb, err } /* Write writes len(p) bytes from p to the underlying data stream. It returns the number of bytes written from p (0 <= n <= len(p)) and any error encountered that caused the write to stop early. */ func (mfb *MultiFileBuffer) Write(output []byte) (int, error) { var b int err := mfb.checkrollover() if err == nil { if mfb.fp == nil { // File existed and can be continued mfb.lock.Lock() mfb.fp, err = os.OpenFile(mfb.basename, os.O_APPEND|os.O_RDWR, 0660) mfb.lock.Unlock() } if err == nil { mfb.lock.Lock() b, err = mfb.fp.Write(output) mfb.lock.Unlock() } } return b, err } /* checkrollover checks if the buffer files should be switched. */ func (mfb *MultiFileBuffer) checkrollover() error { mfb.lock.Lock() defer mfb.lock.Unlock() // Update basename here mfb.basename = mfb.iterator.Basename(mfb.filename) // Rollover if the base file does not exist ex, err := PathExists(mfb.basename) if err == nil && (!ex || mfb.cond.CheckRollover(mfb.basename)) { // Rollover if either the base file does not exist or the // rollover condition is satisfied err = mfb.rollover() } return err } /* Close closes the buffer. */ func (mfb *MultiFileBuffer) Close() error { var err error if mfb.fp != nil { err = mfb.fp.Close() mfb.fp = nil } return err } /* rollover switches the buffer files. */ func (mfb *MultiFileBuffer) rollover() error { var err error // Recursive file renaming function var ensureFileSlot func(fn string) error ensureFileSlot = func(fn string) error { // Check if the file exists already ex, err := PathExists(fn) if ex && err == nil { // Determine new file name newfn := mfb.iterator.NextName(fn) if newfn == "" { // If it is the end of the iteration just delete the file err = os.Remove(fn) } else { // Ensure the new file name is usable err = ensureFileSlot(newfn) // Rename file according to iterator.NextName() if err == nil { err = os.Rename(fn, newfn) } } } return err } // Close existing file err = mfb.Close() // Create file handle if err == nil { err = ensureFileSlot(mfb.basename) if err == nil { // Overwrite existing base file mfb.fp, err = os.OpenFile(mfb.basename, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0660) } } return err } // Rollover conditions // =================== /* RolloverCondition is used by the MultiFileBuffer to check if the buffer files should be switched. */ type RolloverCondition interface { /* CheckRollover checks if the buffer files should be switched. */ CheckRollover(basename string) bool } /* EmptyRolloverCondition creates a rollover condition which is never true. */ func EmptyRolloverCondition() RolloverCondition { return &emptyRolloverCondition{} } /* emptyRolloverCondition is a rollover condition which is never true. */ type emptyRolloverCondition struct { } /* NextName returns the next file name based on the current file name. An empty string means the end of the iteration. */ func (rc *emptyRolloverCondition) CheckRollover(basename string) bool { return false } /* SizeBasedRolloverCondition creates a new rollover condition based on file size. The condition is satisfied if the base file exceeds a certain file size. */ func SizeBasedRolloverCondition(maxSize int64) RolloverCondition { return &sizeBasedRolloverCondition{maxSize} } /* sizeBasedRolloverCondition is the implementation of the size based rollover condition. */ type sizeBasedRolloverCondition struct { maxSize int64 } /* NextName returns the next file name based on the current file name. An empty string means the end of the iteration. */ func (rc *sizeBasedRolloverCondition) CheckRollover(basename string) bool { ret := false if info, err := os.Stat(basename); err == nil { ret = info.Size() >= rc.maxSize } return ret } // FilenameIterator // ================ /* FilenameIterator is used by the MultiFileBuffer to determine the new file name when rotating the buffer files. Basename is called before doing any calculation. This function should do general filename decoration. If the decoration changes over time then the function needs to also handle the cleanup. */ type FilenameIterator interface { /* Basename decorades the initial file name. */ Basename(filename string) string /* NextName returns the next file name based on the current file name. An empty string means the end of the iteration. */ NextName(currentName string) string } /* ConsecutiveNumberIterator creates a new file name iterator which adds numbers at the end of files. Up to maxNum files will be created. A maxNum parameter < 1 means there is no limit. */ func ConsecutiveNumberIterator(maxNum int) FilenameIterator { return &consecutiveNumberIterator{maxNum} } /* consecutiveNumberIterator is the implementation of the consecutive number file iterator. */ type consecutiveNumberIterator struct { maxNum int } /* Basename decorades the initial file name. */ func (it *consecutiveNumberIterator) Basename(filename string) string { return filename } /* NextName returns the next file name based on the current file name. An empty string means the end of the iteration. */ func (it *consecutiveNumberIterator) NextName(currentName string) string { if i := strings.LastIndex(currentName, "."); i > 0 { if num, err := strconv.ParseInt(currentName[i+1:], 10, 64); err == nil { nextNum := int(num + 1) if it.maxNum > 0 && nextNum > it.maxNum { return "" } return fmt.Sprintf("%s.%v", currentName[:i], nextNum) } } return fmt.Sprintf("%s.1", currentName) } /* DailyDateIterator creates a new file name iterator which adds dates at the end of files. The log will be switched at least once every day. Up to maxNumPerDay files will be created per day. A maxNumPerDay parameter < 1 means there is no limit. Up to maxDays different days will be kept (oldest ones are deleted). A maxDays parameter < 1 means everything is kept. */ func DailyDateIterator(maxNumPerDay int, maxDays int) FilenameIterator { return &dailyDateIterator{&consecutiveNumberIterator{maxNumPerDay}, maxDays, timeutil.MakeTimestamp} } /* consecutiveNumberIterator is the implementation of the consecutive number file iterator. */ type dailyDateIterator struct { *consecutiveNumberIterator maxDays int tsFunc func() string // Timestamp function } /* NextName returns the next file name based on the current file name. An empty string means the end of the iteration. */ func (it *dailyDateIterator) Basename(filename string) string { // Get todays date ts := it.tsFunc() today, _ := timeutil.TimestampString(ts, "UTC") today = today[:10] // Cleanup old files if it.maxDays > 0 { prefix := path.Base(filename) dir := path.Dir(filename) if files, err := ioutil.ReadDir(dir); err == nil { var datesToConsider []string // Collect all relevant files foundToday := false for _, f := range files { if strings.HasPrefix(f.Name(), prefix) && len(f.Name()) > len(prefix) { dateString := f.Name()[len(prefix)+1:] if !strings.ContainsRune(dateString, '.') { datesToConsider = append(datesToConsider, dateString) if !foundToday { foundToday = dateString == today } } } } // Make sure today is one of the dates if !foundToday { datesToConsider = append(datesToConsider, today) } // Sort them so the newest ones are kept sort.Strings(datesToConsider) // Check if files need to be removed if len(datesToConsider) > it.maxDays { datesToRemove := datesToConsider[:len(datesToConsider)-it.maxDays] for _, f := range files { for _, dateToRemove := range datesToRemove { if strings.HasPrefix(f.Name(), fmt.Sprintf("%s.%s", prefix, dateToRemove)) { os.Remove(path.Join(dir, f.Name())) } } } } } } return fmt.Sprintf("%s.%s", filename, today) }