failure_case_network_slow.go 2.4 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980
  1. // Copyright 2018 The etcd Authors
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package tester
  15. import (
  16. "time"
  17. "github.com/coreos/etcd/tools/functional-tester/rpcpb"
  18. )
  19. const (
  20. // delay duration to trigger leader election (default election timeout 1s)
  21. triggerElectionDur = 5 * time.Second
  22. // Wait more when it recovers from slow network, because network layer
  23. // needs extra time to propagate traffic control (tc command) change.
  24. // Otherwise, we get different hash values from the previous revision.
  25. // For more detail, please see https://github.com/coreos/etcd/issues/5121.
  26. waitRecover = 5 * time.Second
  27. )
  28. func injectDelayPeerPortTxRx(clus *Cluster, idx int) error {
  29. return clus.sendOperation(idx, rpcpb.Operation_DelayPeerPortTxRx)
  30. }
  31. func recoverDelayPeerPortTxRx(clus *Cluster, idx int) error {
  32. err := clus.sendOperation(idx, rpcpb.Operation_UndelayPeerPortTxRx)
  33. time.Sleep(waitRecover)
  34. return err
  35. }
  36. func newFailureDelayPeerPortTxRxOneFollower(clus *Cluster) Failure {
  37. ff := failureByFunc{
  38. failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ONE_FOLLOWER,
  39. injectMember: injectDelayPeerPortTxRx,
  40. recoverMember: recoverDelayPeerPortTxRx,
  41. }
  42. f := &failureFollower{ff, -1, -1}
  43. return &failureDelay{
  44. Failure: f,
  45. delayDuration: triggerElectionDur,
  46. }
  47. }
  48. func newFailureDelayPeerPortTxRxLeader(clus *Cluster) Failure {
  49. ff := failureByFunc{
  50. failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_LEADER,
  51. injectMember: injectDelayPeerPortTxRx,
  52. recoverMember: recoverDelayPeerPortTxRx,
  53. }
  54. f := &failureLeader{ff, -1, -1}
  55. return &failureDelay{
  56. Failure: f,
  57. delayDuration: triggerElectionDur,
  58. }
  59. }
  60. func newFailureDelayPeerPortTxRxAll(clus *Cluster) Failure {
  61. f := &failureAll{
  62. failureCase: rpcpb.FailureCase_DELAY_PEER_PORT_TX_RX_ALL,
  63. injectMember: injectDelayPeerPortTxRx,
  64. recoverMember: recoverDelayPeerPortTxRx,
  65. }
  66. return &failureDelay{
  67. Failure: f,
  68. delayDuration: triggerElectionDur,
  69. }
  70. }