googlebreaker_test.go 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235
  1. package breaker
  2. import (
  3. "errors"
  4. "math/rand"
  5. "testing"
  6. "time"
  7. "github.com/stretchr/testify/assert"
  8. "github.com/tal-tech/go-zero/core/collection"
  9. "github.com/tal-tech/go-zero/core/mathx"
  10. "github.com/tal-tech/go-zero/core/stat"
  11. )
  12. const (
  13. testBuckets = 10
  14. testInterval = time.Millisecond * 10
  15. )
  16. func init() {
  17. stat.SetReporter(nil)
  18. }
  19. func getGoogleBreaker() *googleBreaker {
  20. st := collection.NewRollingWindow(testBuckets, testInterval)
  21. return &googleBreaker{
  22. stat: st,
  23. k: 5,
  24. proba: mathx.NewProba(),
  25. }
  26. }
  27. func markSuccessWithDuration(b *googleBreaker, count int, sleep time.Duration) {
  28. for i := 0; i < count; i++ {
  29. b.markSuccess()
  30. time.Sleep(sleep)
  31. }
  32. }
  33. func markFailedWithDuration(b *googleBreaker, count int, sleep time.Duration) {
  34. for i := 0; i < count; i++ {
  35. b.markFailure()
  36. time.Sleep(sleep)
  37. }
  38. }
  39. func TestGoogleBreakerClose(t *testing.T) {
  40. b := getGoogleBreaker()
  41. markSuccess(b, 80)
  42. assert.Nil(t, b.accept())
  43. markSuccess(b, 120)
  44. assert.Nil(t, b.accept())
  45. }
  46. func TestGoogleBreakerOpen(t *testing.T) {
  47. b := getGoogleBreaker()
  48. markSuccess(b, 10)
  49. assert.Nil(t, b.accept())
  50. markFailed(b, 100000)
  51. time.Sleep(testInterval * 2)
  52. verify(t, func() bool {
  53. return b.accept() != nil
  54. })
  55. }
  56. func TestGoogleBreakerFallback(t *testing.T) {
  57. b := getGoogleBreaker()
  58. markSuccess(b, 1)
  59. assert.Nil(t, b.accept())
  60. markFailed(b, 10000)
  61. time.Sleep(testInterval * 2)
  62. verify(t, func() bool {
  63. return b.doReq(func() error {
  64. return errors.New("any")
  65. }, func(err error) error {
  66. return nil
  67. }, defaultAcceptable) == nil
  68. })
  69. }
  70. func TestGoogleBreakerReject(t *testing.T) {
  71. b := getGoogleBreaker()
  72. markSuccess(b, 100)
  73. assert.Nil(t, b.accept())
  74. markFailed(b, 10000)
  75. time.Sleep(testInterval)
  76. assert.Equal(t, ErrServiceUnavailable, b.doReq(func() error {
  77. return ErrServiceUnavailable
  78. }, nil, defaultAcceptable))
  79. }
  80. func TestGoogleBreakerAcceptable(t *testing.T) {
  81. b := getGoogleBreaker()
  82. errAcceptable := errors.New("any")
  83. assert.Equal(t, errAcceptable, b.doReq(func() error {
  84. return errAcceptable
  85. }, nil, func(err error) bool {
  86. return err == errAcceptable
  87. }))
  88. }
  89. func TestGoogleBreakerNotAcceptable(t *testing.T) {
  90. b := getGoogleBreaker()
  91. errAcceptable := errors.New("any")
  92. assert.Equal(t, errAcceptable, b.doReq(func() error {
  93. return errAcceptable
  94. }, nil, func(err error) bool {
  95. return err != errAcceptable
  96. }))
  97. }
  98. func TestGoogleBreakerPanic(t *testing.T) {
  99. b := getGoogleBreaker()
  100. assert.Panics(t, func() {
  101. _ = b.doReq(func() error {
  102. panic("fail")
  103. }, nil, defaultAcceptable)
  104. })
  105. }
  106. func TestGoogleBreakerHalfOpen(t *testing.T) {
  107. b := getGoogleBreaker()
  108. assert.Nil(t, b.accept())
  109. t.Run("accept single failed/accept", func(t *testing.T) {
  110. markFailed(b, 10000)
  111. time.Sleep(testInterval * 2)
  112. verify(t, func() bool {
  113. return b.accept() != nil
  114. })
  115. })
  116. t.Run("accept single failed/allow", func(t *testing.T) {
  117. markFailed(b, 10000)
  118. time.Sleep(testInterval * 2)
  119. verify(t, func() bool {
  120. _, err := b.allow()
  121. return err != nil
  122. })
  123. })
  124. time.Sleep(testInterval * testBuckets)
  125. t.Run("accept single succeed", func(t *testing.T) {
  126. assert.Nil(t, b.accept())
  127. markSuccess(b, 10000)
  128. verify(t, func() bool {
  129. return b.accept() == nil
  130. })
  131. })
  132. }
  133. func TestGoogleBreakerSelfProtection(t *testing.T) {
  134. t.Run("total request < 100", func(t *testing.T) {
  135. b := getGoogleBreaker()
  136. markFailed(b, 4)
  137. time.Sleep(testInterval)
  138. assert.Nil(t, b.accept())
  139. })
  140. t.Run("total request > 100, total < 2 * success", func(t *testing.T) {
  141. b := getGoogleBreaker()
  142. size := rand.Intn(10000)
  143. accepts := size + 1
  144. markSuccess(b, accepts)
  145. markFailed(b, size-accepts)
  146. assert.Nil(t, b.accept())
  147. })
  148. }
  149. func TestGoogleBreakerHistory(t *testing.T) {
  150. var b *googleBreaker
  151. var accepts, total int64
  152. sleep := testInterval
  153. t.Run("accepts == total", func(t *testing.T) {
  154. b = getGoogleBreaker()
  155. markSuccessWithDuration(b, 10, sleep/2)
  156. accepts, total = b.history()
  157. assert.Equal(t, int64(10), accepts)
  158. assert.Equal(t, int64(10), total)
  159. })
  160. t.Run("fail == total", func(t *testing.T) {
  161. b = getGoogleBreaker()
  162. markFailedWithDuration(b, 10, sleep/2)
  163. accepts, total = b.history()
  164. assert.Equal(t, int64(0), accepts)
  165. assert.Equal(t, int64(10), total)
  166. })
  167. t.Run("accepts = 1/2 * total, fail = 1/2 * total", func(t *testing.T) {
  168. b = getGoogleBreaker()
  169. markFailedWithDuration(b, 5, sleep/2)
  170. markSuccessWithDuration(b, 5, sleep/2)
  171. accepts, total = b.history()
  172. assert.Equal(t, int64(5), accepts)
  173. assert.Equal(t, int64(10), total)
  174. })
  175. t.Run("auto reset rolling counter", func(t *testing.T) {
  176. b = getGoogleBreaker()
  177. time.Sleep(testInterval * testBuckets)
  178. accepts, total = b.history()
  179. assert.Equal(t, int64(0), accepts)
  180. assert.Equal(t, int64(0), total)
  181. })
  182. }
  183. func BenchmarkGoogleBreakerAllow(b *testing.B) {
  184. breaker := getGoogleBreaker()
  185. b.ResetTimer()
  186. for i := 0; i <= b.N; i++ {
  187. breaker.accept()
  188. if i%2 == 0 {
  189. breaker.markSuccess()
  190. } else {
  191. breaker.markFailure()
  192. }
  193. }
  194. }
  195. func markSuccess(b *googleBreaker, count int) {
  196. for i := 0; i < count; i++ {
  197. p, err := b.allow()
  198. if err != nil {
  199. break
  200. }
  201. p.Accept()
  202. }
  203. }
  204. func markFailed(b *googleBreaker, count int) {
  205. for i := 0; i < count; i++ {
  206. p, err := b.allow()
  207. if err == nil {
  208. p.Reject()
  209. }
  210. }
  211. }