confchange_v2_add_double_auto.txt 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197
  1. # Run a V2 membership change that adds two voters at once and auto-leaves the
  2. # joint configuration. (This is the same as specifying an explicit transition
  3. # since more than one change is being made atomically).
  4. # Bootstrap n1.
  5. add-nodes 1 voters=(1) index=2
  6. ----
  7. INFO 1 switched to configuration voters=(1)
  8. INFO 1 became follower at term 0
  9. INFO newRaft 1 [peers: [1], term: 0, commit: 2, applied: 2, lastindex: 2, lastterm: 1]
  10. campaign 1
  11. ----
  12. INFO 1 is starting a new election at term 0
  13. INFO 1 became candidate at term 1
  14. INFO 1 received MsgVoteResp from 1 at term 1
  15. INFO 1 became leader at term 1
  16. propose-conf-change 1 transition=auto
  17. v2 v3
  18. ----
  19. ok
  20. # Add two "empty" nodes to the cluster, n2 and n3.
  21. add-nodes 2
  22. ----
  23. INFO 2 switched to configuration voters=()
  24. INFO 2 became follower at term 0
  25. INFO newRaft 2 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
  26. INFO 3 switched to configuration voters=()
  27. INFO 3 became follower at term 0
  28. INFO newRaft 3 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
  29. # n1 immediately gets to commit & apply the conf change using only itself. We see that
  30. # it starts transitioning out of that joint configuration (though we will only see that
  31. # proposal in the next ready handling loop, when it is emitted). We also see that this
  32. # is using joint consensus, which it has to since we're carrying out two additions at
  33. # once.
  34. process-ready 1
  35. ----
  36. Ready MustSync=true:
  37. Lead:1 State:StateLeader
  38. HardState Term:1 Vote:1 Commit:4
  39. Entries:
  40. 1/3 EntryNormal ""
  41. 1/4 EntryConfChangeV2 v2 v3
  42. CommittedEntries:
  43. 1/3 EntryNormal ""
  44. 1/4 EntryConfChangeV2 v2 v3
  45. INFO 1 switched to configuration voters=(1 2 3)&&(1) autoleave
  46. INFO initiating automatic transition out of joint configuration voters=(1 2 3)&&(1) autoleave
  47. # n1 immediately probes n2 and n3.
  48. stabilize 1
  49. ----
  50. > 1 handling Ready
  51. Ready MustSync=true:
  52. Entries:
  53. 1/5 EntryConfChangeV2
  54. Messages:
  55. 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2 v3]
  56. 1->3 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2 v3]
  57. # First, play out the whole interaction between n1 and n2. We see n1's probe to
  58. # n2 get rejected (since n2 needs a snapshot); the snapshot is delivered at which
  59. # point n2 switches to the correct config, and n1 catches it up. This notably
  60. # includes the empty conf change which gets committed and applied by both and
  61. # which transitions them out of their joint configuration into the final one (1 2 3).
  62. stabilize 1 2
  63. ----
  64. > 2 receiving messages
  65. 1->2 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2 v3]
  66. INFO 2 [term: 0] received a MsgApp message with higher term from 1 [term: 1]
  67. INFO 2 became follower at term 1
  68. DEBUG 2 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1
  69. > 2 handling Ready
  70. Ready MustSync=true:
  71. Lead:1 State:StateFollower
  72. HardState Term:1 Commit:0
  73. Messages:
  74. 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0)
  75. > 1 receiving messages
  76. 2->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0)
  77. DEBUG 1 received MsgAppResp(MsgApp was rejected, lastindex: 0) from 2 for index 3
  78. DEBUG 1 decreased progress of 2 to [StateProbe match=0 next=1]
  79. DEBUG 1 [firstindex: 3, commit: 4] sent snapshot[index: 4, term: 1] to 2 [StateProbe match=0 next=1]
  80. DEBUG 1 paused sending replication messages to 2 [StateSnapshot match=0 next=1 paused pendingSnap=4]
  81. > 1 handling Ready
  82. Ready MustSync=false:
  83. Messages:
  84. 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true
  85. > 2 receiving messages
  86. 1->2 MsgSnap Term:1 Log:0/0 Snapshot: Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true
  87. INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 4, term: 1]
  88. INFO 2 switched to configuration voters=(1 2 3)&&(1) autoleave
  89. INFO 2 [commit: 4, lastindex: 4, lastterm: 1] restored snapshot [index: 4, term: 1]
  90. INFO 2 [commit: 4] restored snapshot [index: 4, term: 1]
  91. > 2 handling Ready
  92. Ready MustSync=false:
  93. HardState Term:1 Commit:4
  94. Snapshot Index:4 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[1] Learners:[] LearnersNext:[] AutoLeave:true
  95. Messages:
  96. 2->1 MsgAppResp Term:1 Log:0/4
  97. > 1 receiving messages
  98. 2->1 MsgAppResp Term:1 Log:0/4
  99. DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 2 [StateSnapshot match=4 next=5 paused pendingSnap=4]
  100. > 1 handling Ready
  101. Ready MustSync=false:
  102. Messages:
  103. 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2]
  104. > 2 receiving messages
  105. 1->2 MsgApp Term:1 Log:1/4 Commit:4 Entries:[1/5 EntryConfChangeV2]
  106. > 2 handling Ready
  107. Ready MustSync=true:
  108. Entries:
  109. 1/5 EntryConfChangeV2
  110. Messages:
  111. 2->1 MsgAppResp Term:1 Log:0/5
  112. > 1 receiving messages
  113. 2->1 MsgAppResp Term:1 Log:0/5
  114. > 1 handling Ready
  115. Ready MustSync=false:
  116. HardState Term:1 Vote:1 Commit:5
  117. CommittedEntries:
  118. 1/5 EntryConfChangeV2
  119. Messages:
  120. 1->2 MsgApp Term:1 Log:1/5 Commit:5
  121. INFO 1 switched to configuration voters=(1 2 3)
  122. > 2 receiving messages
  123. 1->2 MsgApp Term:1 Log:1/5 Commit:5
  124. > 2 handling Ready
  125. Ready MustSync=false:
  126. HardState Term:1 Commit:5
  127. CommittedEntries:
  128. 1/5 EntryConfChangeV2
  129. Messages:
  130. 2->1 MsgAppResp Term:1 Log:0/5
  131. INFO 2 switched to configuration voters=(1 2 3)
  132. > 1 receiving messages
  133. 2->1 MsgAppResp Term:1 Log:0/5
  134. # n3 immediately receives a snapshot in the final configuration.
  135. stabilize 1 3
  136. ----
  137. > 3 receiving messages
  138. 1->3 MsgApp Term:1 Log:1/3 Commit:4 Entries:[1/4 EntryConfChangeV2 v2 v3]
  139. INFO 3 [term: 0] received a MsgApp message with higher term from 1 [term: 1]
  140. INFO 3 became follower at term 1
  141. DEBUG 3 [logterm: 0, index: 3] rejected MsgApp [logterm: 1, index: 3] from 1
  142. > 3 handling Ready
  143. Ready MustSync=true:
  144. Lead:1 State:StateFollower
  145. HardState Term:1 Commit:0
  146. Messages:
  147. 3->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0)
  148. > 1 receiving messages
  149. 3->1 MsgAppResp Term:1 Log:0/3 Rejected (Hint: 0)
  150. DEBUG 1 received MsgAppResp(MsgApp was rejected, lastindex: 0) from 3 for index 3
  151. DEBUG 1 decreased progress of 3 to [StateProbe match=0 next=1]
  152. DEBUG 1 [firstindex: 3, commit: 5] sent snapshot[index: 5, term: 1] to 3 [StateProbe match=0 next=1]
  153. DEBUG 1 paused sending replication messages to 3 [StateSnapshot match=0 next=1 paused pendingSnap=5]
  154. > 1 handling Ready
  155. Ready MustSync=false:
  156. Messages:
  157. 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false
  158. > 3 receiving messages
  159. 1->3 MsgSnap Term:1 Log:0/0 Snapshot: Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false
  160. INFO log [committed=0, applied=0, unstable.offset=1, len(unstable.Entries)=0] starts to restore snapshot [index: 5, term: 1]
  161. INFO 3 switched to configuration voters=(1 2 3)
  162. INFO 3 [commit: 5, lastindex: 5, lastterm: 1] restored snapshot [index: 5, term: 1]
  163. INFO 3 [commit: 5] restored snapshot [index: 5, term: 1]
  164. > 3 handling Ready
  165. Ready MustSync=false:
  166. HardState Term:1 Commit:5
  167. Snapshot Index:5 Term:1 ConfState:Voters:[1 2 3] VotersOutgoing:[] Learners:[] LearnersNext:[] AutoLeave:false
  168. Messages:
  169. 3->1 MsgAppResp Term:1 Log:0/5
  170. > 1 receiving messages
  171. 3->1 MsgAppResp Term:1 Log:0/5
  172. DEBUG 1 recovered from needing snapshot, resumed sending replication messages to 3 [StateSnapshot match=5 next=6 paused pendingSnap=5]
  173. > 1 handling Ready
  174. Ready MustSync=false:
  175. Messages:
  176. 1->3 MsgApp Term:1 Log:1/5 Commit:5
  177. > 3 receiving messages
  178. 1->3 MsgApp Term:1 Log:1/5 Commit:5
  179. > 3 handling Ready
  180. Ready MustSync=false:
  181. Messages:
  182. 3->1 MsgAppResp Term:1 Log:0/5
  183. > 1 receiving messages
  184. 3->1 MsgAppResp Term:1 Log:0/5
  185. # Nothing else happens.
  186. stabilize
  187. ----
  188. ok