|
@@ -41,8 +41,7 @@ func init() {
|
|
|
rand.Seed(time.Now().UnixNano())
|
|
rand.Seed(time.Now().UnixNano())
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-type SendFunc func(m []raftpb.Message)
|
|
|
|
|
-type SaveFunc func(st raftpb.HardState, ents []raftpb.Entry)
|
|
|
|
|
|
|
+type sendFunc func(m []raftpb.Message)
|
|
|
|
|
|
|
|
type Response struct {
|
|
type Response struct {
|
|
|
Event *store.Event
|
|
Event *store.Event
|
|
@@ -146,18 +145,18 @@ func NewServer(cfg *ServerConfig) *EtcdServer {
|
|
|
cls := NewClusterStore(st, *cfg.Cluster)
|
|
cls := NewClusterStore(st, *cfg.Cluster)
|
|
|
|
|
|
|
|
s := &EtcdServer{
|
|
s := &EtcdServer{
|
|
|
- Store: st,
|
|
|
|
|
- Node: n,
|
|
|
|
|
|
|
+ store: st,
|
|
|
|
|
+ node: n,
|
|
|
name: cfg.Name,
|
|
name: cfg.Name,
|
|
|
- Storage: struct {
|
|
|
|
|
|
|
+ storage: struct {
|
|
|
*wal.WAL
|
|
*wal.WAL
|
|
|
*snap.Snapshotter
|
|
*snap.Snapshotter
|
|
|
}{w, ss},
|
|
}{w, ss},
|
|
|
- Send: Sender(cfg.Transport, cls),
|
|
|
|
|
|
|
+ send: Sender(cfg.Transport, cls),
|
|
|
clientURLs: cfg.ClientURLs,
|
|
clientURLs: cfg.ClientURLs,
|
|
|
- Ticker: time.Tick(100 * time.Millisecond),
|
|
|
|
|
- SyncTicker: time.Tick(500 * time.Millisecond),
|
|
|
|
|
- SnapCount: cfg.SnapCount,
|
|
|
|
|
|
|
+ ticker: time.Tick(100 * time.Millisecond),
|
|
|
|
|
+ syncTicker: time.Tick(500 * time.Millisecond),
|
|
|
|
|
+ snapCount: cfg.SnapCount,
|
|
|
ClusterStore: cls,
|
|
ClusterStore: cls,
|
|
|
}
|
|
}
|
|
|
return s
|
|
return s
|
|
@@ -172,21 +171,21 @@ type EtcdServer struct {
|
|
|
|
|
|
|
|
ClusterStore ClusterStore
|
|
ClusterStore ClusterStore
|
|
|
|
|
|
|
|
- Node raft.Node
|
|
|
|
|
- Store store.Store
|
|
|
|
|
|
|
+ node raft.Node
|
|
|
|
|
+ store store.Store
|
|
|
|
|
|
|
|
- // Send specifies the send function for sending msgs to members. Send
|
|
|
|
|
|
|
+ // send specifies the send function for sending msgs to members. send
|
|
|
// MUST NOT block. It is okay to drop messages, since clients should
|
|
// MUST NOT block. It is okay to drop messages, since clients should
|
|
|
- // timeout and reissue their messages. If Send is nil, server will
|
|
|
|
|
|
|
+ // timeout and reissue their messages. If send is nil, server will
|
|
|
// panic.
|
|
// panic.
|
|
|
- Send SendFunc
|
|
|
|
|
|
|
+ send sendFunc
|
|
|
|
|
|
|
|
- Storage Storage
|
|
|
|
|
|
|
+ storage Storage
|
|
|
|
|
|
|
|
- Ticker <-chan time.Time
|
|
|
|
|
- SyncTicker <-chan time.Time
|
|
|
|
|
|
|
+ ticker <-chan time.Time
|
|
|
|
|
+ syncTicker <-chan time.Time
|
|
|
|
|
|
|
|
- SnapCount int64 // number of entries to trigger a snapshot
|
|
|
|
|
|
|
+ snapCount int64 // number of entries to trigger a snapshot
|
|
|
|
|
|
|
|
// Cache of the latest raft index and raft term the server has seen
|
|
// Cache of the latest raft index and raft term the server has seen
|
|
|
raftIndex int64
|
|
raftIndex int64
|
|
@@ -205,9 +204,9 @@ func (s *EtcdServer) Start() {
|
|
|
// modify a server's fields after it has been sent to Start.
|
|
// modify a server's fields after it has been sent to Start.
|
|
|
// This function is just used for testing.
|
|
// This function is just used for testing.
|
|
|
func (s *EtcdServer) start() {
|
|
func (s *EtcdServer) start() {
|
|
|
- if s.SnapCount == 0 {
|
|
|
|
|
|
|
+ if s.snapCount == 0 {
|
|
|
log.Printf("etcdserver: set snapshot count to default %d", DefaultSnapCount)
|
|
log.Printf("etcdserver: set snapshot count to default %d", DefaultSnapCount)
|
|
|
- s.SnapCount = DefaultSnapCount
|
|
|
|
|
|
|
+ s.snapCount = DefaultSnapCount
|
|
|
}
|
|
}
|
|
|
s.w = wait.New()
|
|
s.w = wait.New()
|
|
|
s.done = make(chan struct{})
|
|
s.done = make(chan struct{})
|
|
@@ -217,7 +216,7 @@ func (s *EtcdServer) start() {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
func (s *EtcdServer) Process(ctx context.Context, m raftpb.Message) error {
|
|
func (s *EtcdServer) Process(ctx context.Context, m raftpb.Message) error {
|
|
|
- return s.Node.Step(ctx, m)
|
|
|
|
|
|
|
+ return s.node.Step(ctx, m)
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
func (s *EtcdServer) run() {
|
|
func (s *EtcdServer) run() {
|
|
@@ -226,12 +225,12 @@ func (s *EtcdServer) run() {
|
|
|
var snapi, appliedi int64
|
|
var snapi, appliedi int64
|
|
|
for {
|
|
for {
|
|
|
select {
|
|
select {
|
|
|
- case <-s.Ticker:
|
|
|
|
|
- s.Node.Tick()
|
|
|
|
|
- case rd := <-s.Node.Ready():
|
|
|
|
|
- s.Storage.Save(rd.HardState, rd.Entries)
|
|
|
|
|
- s.Storage.SaveSnap(rd.Snapshot)
|
|
|
|
|
- s.Send(rd.Messages)
|
|
|
|
|
|
|
+ case <-s.ticker:
|
|
|
|
|
+ s.node.Tick()
|
|
|
|
|
+ case rd := <-s.node.Ready():
|
|
|
|
|
+ s.storage.Save(rd.HardState, rd.Entries)
|
|
|
|
|
+ s.storage.SaveSnap(rd.Snapshot)
|
|
|
|
|
+ s.send(rd.Messages)
|
|
|
|
|
|
|
|
// TODO(bmizerany): do this in the background, but take
|
|
// TODO(bmizerany): do this in the background, but take
|
|
|
// care to apply entries in a single goroutine, and not
|
|
// care to apply entries in a single goroutine, and not
|
|
@@ -250,7 +249,7 @@ func (s *EtcdServer) run() {
|
|
|
if err := cc.Unmarshal(e.Data); err != nil {
|
|
if err := cc.Unmarshal(e.Data); err != nil {
|
|
|
panic("TODO: this is bad, what do we do about it?")
|
|
panic("TODO: this is bad, what do we do about it?")
|
|
|
}
|
|
}
|
|
|
- s.Node.ApplyConfChange(cc)
|
|
|
|
|
|
|
+ s.node.ApplyConfChange(cc)
|
|
|
s.w.Trigger(cc.ID, nil)
|
|
s.w.Trigger(cc.ID, nil)
|
|
|
default:
|
|
default:
|
|
|
panic("unexpected entry type")
|
|
panic("unexpected entry type")
|
|
@@ -266,20 +265,20 @@ func (s *EtcdServer) run() {
|
|
|
|
|
|
|
|
// recover from snapshot if it is more updated than current applied
|
|
// recover from snapshot if it is more updated than current applied
|
|
|
if rd.Snapshot.Index > appliedi {
|
|
if rd.Snapshot.Index > appliedi {
|
|
|
- if err := s.Store.Recovery(rd.Snapshot.Data); err != nil {
|
|
|
|
|
|
|
+ if err := s.store.Recovery(rd.Snapshot.Data); err != nil {
|
|
|
panic("TODO: this is bad, what do we do about it?")
|
|
panic("TODO: this is bad, what do we do about it?")
|
|
|
}
|
|
}
|
|
|
appliedi = rd.Snapshot.Index
|
|
appliedi = rd.Snapshot.Index
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- if appliedi-snapi > s.SnapCount {
|
|
|
|
|
|
|
+ if appliedi-snapi > s.snapCount {
|
|
|
s.snapshot()
|
|
s.snapshot()
|
|
|
snapi = appliedi
|
|
snapi = appliedi
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
if rd.SoftState != nil {
|
|
if rd.SoftState != nil {
|
|
|
if rd.RaftState == raft.StateLeader {
|
|
if rd.RaftState == raft.StateLeader {
|
|
|
- syncC = s.SyncTicker
|
|
|
|
|
|
|
+ syncC = s.syncTicker
|
|
|
} else {
|
|
} else {
|
|
|
syncC = nil
|
|
syncC = nil
|
|
|
}
|
|
}
|
|
@@ -299,11 +298,11 @@ func (s *EtcdServer) run() {
|
|
|
// Stop stops the server, and shuts down the running goroutine. Stop should be
|
|
// Stop stops the server, and shuts down the running goroutine. Stop should be
|
|
|
// called after a Start(s), otherwise it will block forever.
|
|
// called after a Start(s), otherwise it will block forever.
|
|
|
func (s *EtcdServer) Stop() {
|
|
func (s *EtcdServer) Stop() {
|
|
|
- s.Node.Stop()
|
|
|
|
|
|
|
+ s.node.Stop()
|
|
|
close(s.done)
|
|
close(s.done)
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
-// Do interprets r and performs an operation on s.Store according to r.Method
|
|
|
|
|
|
|
+// Do interprets r and performs an operation on s.store according to r.Method
|
|
|
// and other fields. If r.Method is "POST", "PUT", "DELETE", or a "GET" with
|
|
// and other fields. If r.Method is "POST", "PUT", "DELETE", or a "GET" with
|
|
|
// Quorum == true, r will be sent through consensus before performing its
|
|
// Quorum == true, r will be sent through consensus before performing its
|
|
|
// respective operation. Do will block until an action is performed or there is
|
|
// respective operation. Do will block until an action is performed or there is
|
|
@@ -322,7 +321,7 @@ func (s *EtcdServer) Do(ctx context.Context, r pb.Request) (Response, error) {
|
|
|
return Response{}, err
|
|
return Response{}, err
|
|
|
}
|
|
}
|
|
|
ch := s.w.Register(r.ID)
|
|
ch := s.w.Register(r.ID)
|
|
|
- s.Node.Propose(ctx, data)
|
|
|
|
|
|
|
+ s.node.Propose(ctx, data)
|
|
|
select {
|
|
select {
|
|
|
case x := <-ch:
|
|
case x := <-ch:
|
|
|
resp := x.(Response)
|
|
resp := x.(Response)
|
|
@@ -336,13 +335,13 @@ func (s *EtcdServer) Do(ctx context.Context, r pb.Request) (Response, error) {
|
|
|
case "GET":
|
|
case "GET":
|
|
|
switch {
|
|
switch {
|
|
|
case r.Wait:
|
|
case r.Wait:
|
|
|
- wc, err := s.Store.Watch(r.Path, r.Recursive, r.Stream, r.Since)
|
|
|
|
|
|
|
+ wc, err := s.store.Watch(r.Path, r.Recursive, r.Stream, r.Since)
|
|
|
if err != nil {
|
|
if err != nil {
|
|
|
return Response{}, err
|
|
return Response{}, err
|
|
|
}
|
|
}
|
|
|
return Response{Watcher: wc}, nil
|
|
return Response{Watcher: wc}, nil
|
|
|
default:
|
|
default:
|
|
|
- ev, err := s.Store.Get(r.Path, r.Recursive, r.Sorted)
|
|
|
|
|
|
|
+ ev, err := s.store.Get(r.Path, r.Recursive, r.Sorted)
|
|
|
if err != nil {
|
|
if err != nil {
|
|
|
return Response{}, err
|
|
return Response{}, err
|
|
|
}
|
|
}
|
|
@@ -385,7 +384,7 @@ func (s *EtcdServer) Term() int64 {
|
|
|
// It will block until the change is performed or there is an error.
|
|
// It will block until the change is performed or there is an error.
|
|
|
func (s *EtcdServer) configure(ctx context.Context, cc raftpb.ConfChange) error {
|
|
func (s *EtcdServer) configure(ctx context.Context, cc raftpb.ConfChange) error {
|
|
|
ch := s.w.Register(cc.ID)
|
|
ch := s.w.Register(cc.ID)
|
|
|
- if err := s.Node.ProposeConfChange(ctx, cc); err != nil {
|
|
|
|
|
|
|
+ if err := s.node.ProposeConfChange(ctx, cc); err != nil {
|
|
|
log.Printf("configure error: %v", err)
|
|
log.Printf("configure error: %v", err)
|
|
|
s.w.Trigger(cc.ID, nil)
|
|
s.w.Trigger(cc.ID, nil)
|
|
|
return err
|
|
return err
|
|
@@ -419,7 +418,7 @@ func (s *EtcdServer) sync(timeout time.Duration) {
|
|
|
// There is no promise that node has leader when do SYNC request,
|
|
// There is no promise that node has leader when do SYNC request,
|
|
|
// so it uses goroutine to propose.
|
|
// so it uses goroutine to propose.
|
|
|
go func() {
|
|
go func() {
|
|
|
- s.Node.Propose(ctx, data)
|
|
|
|
|
|
|
+ s.node.Propose(ctx, data)
|
|
|
cancel()
|
|
cancel()
|
|
|
}()
|
|
}()
|
|
|
}
|
|
}
|
|
@@ -479,31 +478,31 @@ func (s *EtcdServer) apply(r pb.Request) Response {
|
|
|
expr := getExpirationTime(&r)
|
|
expr := getExpirationTime(&r)
|
|
|
switch r.Method {
|
|
switch r.Method {
|
|
|
case "POST":
|
|
case "POST":
|
|
|
- return f(s.Store.Create(r.Path, r.Dir, r.Val, true, expr))
|
|
|
|
|
|
|
+ return f(s.store.Create(r.Path, r.Dir, r.Val, true, expr))
|
|
|
case "PUT":
|
|
case "PUT":
|
|
|
exists, existsSet := getBool(r.PrevExist)
|
|
exists, existsSet := getBool(r.PrevExist)
|
|
|
switch {
|
|
switch {
|
|
|
case existsSet:
|
|
case existsSet:
|
|
|
if exists {
|
|
if exists {
|
|
|
- return f(s.Store.Update(r.Path, r.Val, expr))
|
|
|
|
|
|
|
+ return f(s.store.Update(r.Path, r.Val, expr))
|
|
|
}
|
|
}
|
|
|
- return f(s.Store.Create(r.Path, r.Dir, r.Val, false, expr))
|
|
|
|
|
|
|
+ return f(s.store.Create(r.Path, r.Dir, r.Val, false, expr))
|
|
|
case r.PrevIndex > 0 || r.PrevValue != "":
|
|
case r.PrevIndex > 0 || r.PrevValue != "":
|
|
|
- return f(s.Store.CompareAndSwap(r.Path, r.PrevValue, r.PrevIndex, r.Val, expr))
|
|
|
|
|
|
|
+ return f(s.store.CompareAndSwap(r.Path, r.PrevValue, r.PrevIndex, r.Val, expr))
|
|
|
default:
|
|
default:
|
|
|
- return f(s.Store.Set(r.Path, r.Dir, r.Val, expr))
|
|
|
|
|
|
|
+ return f(s.store.Set(r.Path, r.Dir, r.Val, expr))
|
|
|
}
|
|
}
|
|
|
case "DELETE":
|
|
case "DELETE":
|
|
|
switch {
|
|
switch {
|
|
|
case r.PrevIndex > 0 || r.PrevValue != "":
|
|
case r.PrevIndex > 0 || r.PrevValue != "":
|
|
|
- return f(s.Store.CompareAndDelete(r.Path, r.PrevValue, r.PrevIndex))
|
|
|
|
|
|
|
+ return f(s.store.CompareAndDelete(r.Path, r.PrevValue, r.PrevIndex))
|
|
|
default:
|
|
default:
|
|
|
- return f(s.Store.Delete(r.Path, r.Dir, r.Recursive))
|
|
|
|
|
|
|
+ return f(s.store.Delete(r.Path, r.Dir, r.Recursive))
|
|
|
}
|
|
}
|
|
|
case "QGET":
|
|
case "QGET":
|
|
|
- return f(s.Store.Get(r.Path, r.Recursive, r.Sorted))
|
|
|
|
|
|
|
+ return f(s.store.Get(r.Path, r.Recursive, r.Sorted))
|
|
|
case "SYNC":
|
|
case "SYNC":
|
|
|
- s.Store.DeleteExpiredKeys(time.Unix(0, r.Time))
|
|
|
|
|
|
|
+ s.store.DeleteExpiredKeys(time.Unix(0, r.Time))
|
|
|
return Response{}
|
|
return Response{}
|
|
|
default:
|
|
default:
|
|
|
// This should never be reached, but just in case:
|
|
// This should never be reached, but just in case:
|
|
@@ -513,14 +512,14 @@ func (s *EtcdServer) apply(r pb.Request) Response {
|
|
|
|
|
|
|
|
// TODO: non-blocking snapshot
|
|
// TODO: non-blocking snapshot
|
|
|
func (s *EtcdServer) snapshot() {
|
|
func (s *EtcdServer) snapshot() {
|
|
|
- d, err := s.Store.Save()
|
|
|
|
|
|
|
+ d, err := s.store.Save()
|
|
|
// TODO: current store will never fail to do a snapshot
|
|
// TODO: current store will never fail to do a snapshot
|
|
|
// what should we do if the store might fail?
|
|
// what should we do if the store might fail?
|
|
|
if err != nil {
|
|
if err != nil {
|
|
|
panic("TODO: this is bad, what do we do about it?")
|
|
panic("TODO: this is bad, what do we do about it?")
|
|
|
}
|
|
}
|
|
|
- s.Node.Compact(d)
|
|
|
|
|
- s.Storage.Cut()
|
|
|
|
|
|
|
+ s.node.Compact(d)
|
|
|
|
|
+ s.storage.Cut()
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
// TODO: move the function to /id pkg maybe?
|
|
// TODO: move the function to /id pkg maybe?
|