faults.sh 1.5 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465
  1. #!/bin/bash
  2. PROCFILE="tools/local-tester/Procfile"
  3. function wait_time {
  4. expr $RANDOM % 10 + 1
  5. }
  6. function cycle {
  7. for a; do
  8. echo "cycling $a"
  9. goreman -f $PROCFILE run stop $a || echo "could not stop $a"
  10. sleep `wait_time`s
  11. goreman -f $PROCFILE run restart $a || echo "could not restart $a"
  12. done
  13. }
  14. function cycle_members {
  15. cycle etcd1 etcd2 etcd3
  16. }
  17. function cycle_pbridge {
  18. cycle pbridge1 pbridge2 pbridge3
  19. }
  20. function cycle_cbridge {
  21. cycle cbridge1 cbridge2 cbridge3
  22. }
  23. function cycle_stresser {
  24. cycle stress-put
  25. }
  26. function kill_maj {
  27. idx="etcd"`expr $RANDOM % 3 + 1`
  28. idx2="$idx"
  29. while [ "$idx" == "$idx2" ]; do
  30. idx2="etcd"`expr $RANDOM % 3 + 1`
  31. done
  32. echo "kill majority $idx $idx2"
  33. goreman -f $PROCFILE run stop $idx || echo "could not stop $idx"
  34. goreman -f $PROCFILE run stop $idx2 || echo "could not stop $idx2"
  35. sleep `wait_time`s
  36. goreman -f $PROCFILE run restart $idx || echo "could not restart $idx"
  37. goreman -f $PROCFILE run restart $idx2 || echo "could not restart $idx2"
  38. }
  39. function kill_all {
  40. for a in etcd1 etcd2 etcd3; do
  41. goreman -f $PROCFILE run stop $a || echo "could not stop $a"
  42. done
  43. sleep `wait_time`s
  44. for a in etcd1 etcd2 etcd3; do
  45. goreman -f $PROCFILE run restart $a || echo "could not restart $a"
  46. done
  47. }
  48. function choose {
  49. faults=(cycle_members kill_maj kill_all cycle_pbridge cycle_cbridge cycle_stresser)
  50. fault=${faults[`expr $RANDOM % ${#faults[@]}`]}
  51. echo $fault
  52. $fault || echo "failed: $fault"
  53. }
  54. sleep 2s
  55. while [ 1 ]; do
  56. choose
  57. done