rotatelogger.go 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328
  1. package logx
  2. import (
  3. "compress/gzip"
  4. "errors"
  5. "fmt"
  6. "io"
  7. "log"
  8. "os"
  9. "path"
  10. "path/filepath"
  11. "strings"
  12. "sync"
  13. "time"
  14. "github.com/tal-tech/go-zero/core/fs"
  15. "github.com/tal-tech/go-zero/core/lang"
  16. "github.com/tal-tech/go-zero/core/timex"
  17. )
  18. const (
  19. dateFormat = "2006-01-02"
  20. hoursPerDay = 24
  21. bufferSize = 100
  22. defaultDirMode = 0755
  23. defaultFileMode = 0600
  24. )
  25. // ErrLogFileClosed is an error that indicates the log file is already closed.
  26. var ErrLogFileClosed = errors.New("error: log file closed")
  27. type (
  28. // A RotateRule interface is used to define the log rotating rules.
  29. RotateRule interface {
  30. BackupFileName() string
  31. MarkRotated()
  32. OutdatedFiles() []string
  33. ShallRotate() bool
  34. }
  35. // A RotateLogger is a Logger that can rotate log files with given rules.
  36. RotateLogger struct {
  37. filename string
  38. backup string
  39. fp *os.File
  40. channel chan []byte
  41. done chan lang.PlaceholderType
  42. rule RotateRule
  43. compress bool
  44. keepDays int
  45. // can't use threading.RoutineGroup because of cycle import
  46. waitGroup sync.WaitGroup
  47. closeOnce sync.Once
  48. }
  49. // A DailyRotateRule is a rule to daily rotate the log files.
  50. DailyRotateRule struct {
  51. rotatedTime string
  52. filename string
  53. delimiter string
  54. days int
  55. gzip bool
  56. }
  57. )
  58. // DefaultRotateRule is a default log rotating rule, currently DailyRotateRule.
  59. func DefaultRotateRule(filename, delimiter string, days int, gzip bool) RotateRule {
  60. return &DailyRotateRule{
  61. rotatedTime: getNowDate(),
  62. filename: filename,
  63. delimiter: delimiter,
  64. days: days,
  65. gzip: gzip,
  66. }
  67. }
  68. // BackupFileName returns the backup filename on rotating.
  69. func (r *DailyRotateRule) BackupFileName() string {
  70. return fmt.Sprintf("%s%s%s", r.filename, r.delimiter, getNowDate())
  71. }
  72. // MarkRotated marks the rotated time of r to be the current time.
  73. func (r *DailyRotateRule) MarkRotated() {
  74. r.rotatedTime = getNowDate()
  75. }
  76. // OutdatedFiles returns the files that exceeded the keeping days.
  77. func (r *DailyRotateRule) OutdatedFiles() []string {
  78. if r.days <= 0 {
  79. return nil
  80. }
  81. var pattern string
  82. if r.gzip {
  83. pattern = fmt.Sprintf("%s%s*.gz", r.filename, r.delimiter)
  84. } else {
  85. pattern = fmt.Sprintf("%s%s*", r.filename, r.delimiter)
  86. }
  87. files, err := filepath.Glob(pattern)
  88. if err != nil {
  89. Errorf("failed to delete outdated log files, error: %s", err)
  90. return nil
  91. }
  92. var buf strings.Builder
  93. boundary := time.Now().Add(-time.Hour * time.Duration(hoursPerDay*r.days)).Format(dateFormat)
  94. fmt.Fprintf(&buf, "%s%s%s", r.filename, r.delimiter, boundary)
  95. if r.gzip {
  96. buf.WriteString(".gz")
  97. }
  98. boundaryFile := buf.String()
  99. var outdates []string
  100. for _, file := range files {
  101. if file < boundaryFile {
  102. outdates = append(outdates, file)
  103. }
  104. }
  105. return outdates
  106. }
  107. // ShallRotate checks if the file should be rotated.
  108. func (r *DailyRotateRule) ShallRotate() bool {
  109. return len(r.rotatedTime) > 0 && getNowDate() != r.rotatedTime
  110. }
  111. // NewLogger returns a RotateLogger with given filename and rule, etc.
  112. func NewLogger(filename string, rule RotateRule, compress bool) (*RotateLogger, error) {
  113. l := &RotateLogger{
  114. filename: filename,
  115. channel: make(chan []byte, bufferSize),
  116. done: make(chan lang.PlaceholderType),
  117. rule: rule,
  118. compress: compress,
  119. }
  120. if err := l.init(); err != nil {
  121. return nil, err
  122. }
  123. l.startWorker()
  124. return l, nil
  125. }
  126. // Close closes l.
  127. func (l *RotateLogger) Close() error {
  128. var err error
  129. l.closeOnce.Do(func() {
  130. close(l.done)
  131. l.waitGroup.Wait()
  132. if err = l.fp.Sync(); err != nil {
  133. return
  134. }
  135. err = l.fp.Close()
  136. })
  137. return err
  138. }
  139. func (l *RotateLogger) Write(data []byte) (int, error) {
  140. select {
  141. case l.channel <- data:
  142. return len(data), nil
  143. case <-l.done:
  144. log.Println(string(data))
  145. return 0, ErrLogFileClosed
  146. }
  147. }
  148. func (l *RotateLogger) getBackupFilename() string {
  149. if len(l.backup) == 0 {
  150. return l.rule.BackupFileName()
  151. }
  152. return l.backup
  153. }
  154. func (l *RotateLogger) init() error {
  155. l.backup = l.rule.BackupFileName()
  156. if _, err := os.Stat(l.filename); err != nil {
  157. basePath := path.Dir(l.filename)
  158. if _, err = os.Stat(basePath); err != nil {
  159. if err = os.MkdirAll(basePath, defaultDirMode); err != nil {
  160. return err
  161. }
  162. }
  163. if l.fp, err = os.Create(l.filename); err != nil {
  164. return err
  165. }
  166. } else if l.fp, err = os.OpenFile(l.filename, os.O_APPEND|os.O_WRONLY, defaultFileMode); err != nil {
  167. return err
  168. }
  169. fs.CloseOnExec(l.fp)
  170. return nil
  171. }
  172. func (l *RotateLogger) maybeCompressFile(file string) {
  173. if !l.compress {
  174. return
  175. }
  176. defer func() {
  177. if r := recover(); r != nil {
  178. ErrorStack(r)
  179. }
  180. }()
  181. compressLogFile(file)
  182. }
  183. func (l *RotateLogger) maybeDeleteOutdatedFiles() {
  184. files := l.rule.OutdatedFiles()
  185. for _, file := range files {
  186. if err := os.Remove(file); err != nil {
  187. Errorf("failed to remove outdated file: %s", file)
  188. }
  189. }
  190. }
  191. func (l *RotateLogger) postRotate(file string) {
  192. go func() {
  193. // we cannot use threading.GoSafe here, because of import cycle.
  194. l.maybeCompressFile(file)
  195. l.maybeDeleteOutdatedFiles()
  196. }()
  197. }
  198. func (l *RotateLogger) rotate() error {
  199. if l.fp != nil {
  200. err := l.fp.Close()
  201. l.fp = nil
  202. if err != nil {
  203. return err
  204. }
  205. }
  206. _, err := os.Stat(l.filename)
  207. if err == nil && len(l.backup) > 0 {
  208. backupFilename := l.getBackupFilename()
  209. err = os.Rename(l.filename, backupFilename)
  210. if err != nil {
  211. return err
  212. }
  213. l.postRotate(backupFilename)
  214. }
  215. l.backup = l.rule.BackupFileName()
  216. if l.fp, err = os.Create(l.filename); err == nil {
  217. fs.CloseOnExec(l.fp)
  218. }
  219. return err
  220. }
  221. func (l *RotateLogger) startWorker() {
  222. l.waitGroup.Add(1)
  223. go func() {
  224. defer l.waitGroup.Done()
  225. for {
  226. select {
  227. case event := <-l.channel:
  228. l.write(event)
  229. case <-l.done:
  230. return
  231. }
  232. }
  233. }()
  234. }
  235. func (l *RotateLogger) write(v []byte) {
  236. if l.rule.ShallRotate() {
  237. if err := l.rotate(); err != nil {
  238. log.Println(err)
  239. } else {
  240. l.rule.MarkRotated()
  241. }
  242. }
  243. if l.fp != nil {
  244. l.fp.Write(v)
  245. }
  246. }
  247. func compressLogFile(file string) {
  248. start := timex.Now()
  249. Infof("compressing log file: %s", file)
  250. if err := gzipFile(file); err != nil {
  251. Errorf("compress error: %s", err)
  252. } else {
  253. Infof("compressed log file: %s, took %s", file, timex.Since(start))
  254. }
  255. }
  256. func getNowDate() string {
  257. return time.Now().Format(dateFormat)
  258. }
  259. func gzipFile(file string) error {
  260. in, err := os.Open(file)
  261. if err != nil {
  262. return err
  263. }
  264. defer in.Close()
  265. out, err := os.Create(fmt.Sprintf("%s.gz", file))
  266. if err != nil {
  267. return err
  268. }
  269. defer out.Close()
  270. w := gzip.NewWriter(out)
  271. if _, err = io.Copy(w, in); err != nil {
  272. return err
  273. } else if err = w.Close(); err != nil {
  274. return err
  275. }
  276. return os.Remove(file)
  277. }