metrics.go 5.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226
  1. package stat
  2. import (
  3. "os"
  4. "sync"
  5. "time"
  6. "github.com/tal-tech/go-zero/core/executors"
  7. "github.com/tal-tech/go-zero/core/logx"
  8. "github.com/tal-tech/go-zero/core/syncx"
  9. )
  10. var (
  11. logInterval = time.Minute
  12. writerLock sync.Mutex
  13. reportWriter Writer = nil
  14. logEnabled = syncx.ForAtomicBool(true)
  15. )
  16. type (
  17. // Writer interface wraps the Write method.
  18. Writer interface {
  19. Write(report *StatReport) error
  20. }
  21. // A StatReport is a stat report entry.
  22. StatReport struct {
  23. Name string `json:"name"`
  24. Timestamp int64 `json:"tm"`
  25. Pid int `json:"pid"`
  26. ReqsPerSecond float32 `json:"qps"`
  27. Drops int `json:"drops"`
  28. Average float32 `json:"avg"`
  29. Median float32 `json:"med"`
  30. Top90th float32 `json:"t90"`
  31. Top99th float32 `json:"t99"`
  32. Top99p9th float32 `json:"t99p9"`
  33. }
  34. // A Metrics is used to log and report stat reports.
  35. Metrics struct {
  36. executor *executors.PeriodicalExecutor
  37. container *metricsContainer
  38. }
  39. )
  40. // DisableLog disables logs of stats.
  41. func DisableLog() {
  42. logEnabled.Set(false)
  43. }
  44. // SetReportWriter sets the report writer.
  45. func SetReportWriter(writer Writer) {
  46. writerLock.Lock()
  47. reportWriter = writer
  48. writerLock.Unlock()
  49. }
  50. // NewMetrics returns a Metrics.
  51. func NewMetrics(name string) *Metrics {
  52. container := &metricsContainer{
  53. name: name,
  54. pid: os.Getpid(),
  55. }
  56. return &Metrics{
  57. executor: executors.NewPeriodicalExecutor(logInterval, container),
  58. container: container,
  59. }
  60. }
  61. // Add adds task to m.
  62. func (m *Metrics) Add(task Task) {
  63. m.executor.Add(task)
  64. }
  65. // AddDrop adds a drop to m.
  66. func (m *Metrics) AddDrop() {
  67. m.executor.Add(Task{
  68. Drop: true,
  69. })
  70. }
  71. // SetName sets the name of m.
  72. func (m *Metrics) SetName(name string) {
  73. m.executor.Sync(func() {
  74. m.container.name = name
  75. })
  76. }
  77. type (
  78. tasksDurationPair struct {
  79. tasks []Task
  80. duration time.Duration
  81. drops int
  82. }
  83. metricsContainer struct {
  84. name string
  85. pid int
  86. tasks []Task
  87. duration time.Duration
  88. drops int
  89. }
  90. )
  91. func (c *metricsContainer) AddTask(v interface{}) bool {
  92. if task, ok := v.(Task); ok {
  93. if task.Drop {
  94. c.drops++
  95. } else {
  96. c.tasks = append(c.tasks, task)
  97. c.duration += task.Duration
  98. }
  99. }
  100. return false
  101. }
  102. func (c *metricsContainer) Execute(v interface{}) {
  103. pair := v.(tasksDurationPair)
  104. tasks := pair.tasks
  105. duration := pair.duration
  106. drops := pair.drops
  107. size := len(tasks)
  108. report := &StatReport{
  109. Name: c.name,
  110. Timestamp: time.Now().Unix(),
  111. Pid: c.pid,
  112. ReqsPerSecond: float32(size) / float32(logInterval/time.Second),
  113. Drops: drops,
  114. }
  115. if size > 0 {
  116. report.Average = float32(duration/time.Millisecond) / float32(size)
  117. fiftyPercent := size >> 1
  118. if fiftyPercent > 0 {
  119. top50pTasks := topK(tasks, fiftyPercent)
  120. medianTask := top50pTasks[0]
  121. report.Median = float32(medianTask.Duration) / float32(time.Millisecond)
  122. tenPercent := fiftyPercent / 5
  123. if tenPercent > 0 {
  124. top10pTasks := topK(tasks, tenPercent)
  125. task90th := top10pTasks[0]
  126. report.Top90th = float32(task90th.Duration) / float32(time.Millisecond)
  127. onePercent := tenPercent / 10
  128. if onePercent > 0 {
  129. top1pTasks := topK(top10pTasks, onePercent)
  130. task99th := top1pTasks[0]
  131. report.Top99th = float32(task99th.Duration) / float32(time.Millisecond)
  132. pointOnePercent := onePercent / 10
  133. if pointOnePercent > 0 {
  134. topPointOneTasks := topK(top1pTasks, pointOnePercent)
  135. task99Point9th := topPointOneTasks[0]
  136. report.Top99p9th = float32(task99Point9th.Duration) / float32(time.Millisecond)
  137. } else {
  138. report.Top99p9th = getTopDuration(top1pTasks)
  139. }
  140. } else {
  141. mostDuration := getTopDuration(top10pTasks)
  142. report.Top99th = mostDuration
  143. report.Top99p9th = mostDuration
  144. }
  145. } else {
  146. mostDuration := getTopDuration(tasks)
  147. report.Top90th = mostDuration
  148. report.Top99th = mostDuration
  149. report.Top99p9th = mostDuration
  150. }
  151. } else {
  152. mostDuration := getTopDuration(tasks)
  153. report.Median = mostDuration
  154. report.Top90th = mostDuration
  155. report.Top99th = mostDuration
  156. report.Top99p9th = mostDuration
  157. }
  158. }
  159. log(report)
  160. }
  161. func (c *metricsContainer) RemoveAll() interface{} {
  162. tasks := c.tasks
  163. duration := c.duration
  164. drops := c.drops
  165. c.tasks = nil
  166. c.duration = 0
  167. c.drops = 0
  168. return tasksDurationPair{
  169. tasks: tasks,
  170. duration: duration,
  171. drops: drops,
  172. }
  173. }
  174. func getTopDuration(tasks []Task) float32 {
  175. top := topK(tasks, 1)
  176. if len(top) < 1 {
  177. return 0
  178. }
  179. return float32(top[0].Duration) / float32(time.Millisecond)
  180. }
  181. func log(report *StatReport) {
  182. writeReport(report)
  183. if logEnabled.True() {
  184. logx.Statf("(%s) - qps: %.1f/s, drops: %d, avg time: %.1fms, med: %.1fms, "+
  185. "90th: %.1fms, 99th: %.1fms, 99.9th: %.1fms",
  186. report.Name, report.ReqsPerSecond, report.Drops, report.Average, report.Median,
  187. report.Top90th, report.Top99th, report.Top99p9th)
  188. }
  189. }
  190. func writeReport(report *StatReport) {
  191. writerLock.Lock()
  192. defer writerLock.Unlock()
  193. if reportWriter != nil {
  194. if err := reportWriter.Write(report); err != nil {
  195. logx.Error(err)
  196. }
  197. }
  198. }