Merge pull request #5793 from mitake/auth-revision

auth, etcdserver: introduce revision of authStore for avoiding TOCTOU problem

auth/simple_token.go

@@ -20,6 +20,7 @@ package auth
 import (
 	"crypto/rand"
 	"math/big"
+	"strings"
 )
 
 const (
@@ -53,3 +54,14 @@ func (as *authStore) assignSimpleTokenToUser(username, token string) {
 	as.simpleTokens[token] = username
 	as.simpleTokensMu.Unlock()
 }
+
+func (as *authStore) invalidateUser(username string) {
+	as.simpleTokensMu.Lock()
+	defer as.simpleTokensMu.Unlock()
+
+	for token, name := range as.simpleTokens {
+		if strings.Compare(name, username) == 0 {
+			delete(as.simpleTokens, token)
+		}
+	}
+}

auth/store.go

@@ -16,6 +16,7 @@ package auth
 
 import (
 	"bytes"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"sort"
@@ -35,6 +36,8 @@ var (
 	authEnabled   = []byte{1}
 	authDisabled  = []byte{0}
 
+	revisionKey = []byte("authRevision")
+
 	authBucketName      = []byte("auth")
 	authUsersBucketName = []byte("authUsers")
 	authRolesBucketName = []byte("authRoles")
@@ -52,6 +55,7 @@ var (
 	ErrRoleNotGranted       = errors.New("auth: role is not granted to the user")
 	ErrPermissionNotGranted = errors.New("auth: permission is not granted to the role")
 	ErrAuthNotEnabled       = errors.New("auth: authentication is not enabled")
+	ErrAuthOldRevision      = errors.New("auth: revision in header is old")
 
 	// BcryptCost is the algorithm cost / strength for hashing auth passwords
 	BcryptCost = bcrypt.DefaultCost
@@ -60,8 +64,15 @@ var (
 const (
 	rootUser = "root"
 	rootRole = "root"
+
+	revBytesLen = 8
 )
 
+type AuthInfo struct {
+	Username string
+	Revision uint64
+}
+
 type AuthStore interface {
 	// AuthEnable turns on the authentication feature
 	AuthEnable() error
@@ -114,23 +125,27 @@ type AuthStore interface {
 	// RoleList gets a list of all roles
 	RoleList(r *pb.AuthRoleListRequest) (*pb.AuthRoleListResponse, error)
 
-	// UsernameFromToken gets a username from the given Token
-	UsernameFromToken(token string) (string, bool)
+	// AuthInfoFromToken gets a username from the given Token and current revision number
+	// (The revision number is used for preventing the TOCTOU problem)
+	AuthInfoFromToken(token string) (*AuthInfo, bool)
 
 	// IsPutPermitted checks put permission of the user
-	IsPutPermitted(username string, key []byte) bool
+	IsPutPermitted(authInfo *AuthInfo, key []byte) error
 
 	// IsRangePermitted checks range permission of the user
-	IsRangePermitted(username string, key, rangeEnd []byte) bool
+	IsRangePermitted(authInfo *AuthInfo, key, rangeEnd []byte) error
 
 	// IsDeleteRangePermitted checks delete-range permission of the user
-	IsDeleteRangePermitted(username string, key, rangeEnd []byte) bool
+	IsDeleteRangePermitted(authInfo *AuthInfo, key, rangeEnd []byte) error
 
 	// IsAdminPermitted checks admin permission of the user
-	IsAdminPermitted(username string) bool
+	IsAdminPermitted(authInfo *AuthInfo) error
 
 	// GenSimpleToken produces a simple random string
 	GenSimpleToken() (string, error)
+
+	// Revision gets current revision of authStore
+	Revision() uint64
 }
 
 type authStore struct {
@@ -142,6 +157,8 @@ type authStore struct {
 
 	simpleTokensMu sync.RWMutex
 	simpleTokens   map[string]string // token -> username
+
+	revision uint64
 }
 
 func (as *authStore) AuthEnable() error {
@@ -170,6 +187,8 @@ func (as *authStore) AuthEnable() error {
 
 	as.rangePermCache = make(map[string]*unifiedRangePermissions)
 
+	as.revision = getRevision(tx)
+
 	plog.Noticef("Authentication enabled")
 
 	return nil
@@ -180,6 +199,7 @@ func (as *authStore) AuthDisable() {
 	tx := b.BatchTx()
 	tx.Lock()
 	tx.UnsafePut(authBucketName, enableFlagKey, authDisabled)
+	as.commitRevision(tx)
 	tx.Unlock()
 	b.ForceCommit()
 
@@ -235,6 +255,9 @@ func (as *authStore) Recover(be backend.Backend) {
 			enabled = true
 		}
 	}
+
+	as.revision = getRevision(tx)
+
 	tx.Unlock()
 
 	as.enabledMu.Lock()
@@ -265,6 +288,8 @@ func (as *authStore) UserAdd(r *pb.AuthUserAddRequest) (*pb.AuthUserAddResponse,
 
 	putUser(tx, newUser)
 
+	as.commitRevision(tx)
+
 	plog.Noticef("added a new user: %s", r.Name)
 
 	return &pb.AuthUserAddResponse{}, nil
@@ -282,6 +307,11 @@ func (as *authStore) UserDelete(r *pb.AuthUserDeleteRequest) (*pb.AuthUserDelete
 
 	delUser(tx, r.Name)
 
+	as.commitRevision(tx)
+
+	as.invalidateCachedPerm(r.Name)
+	as.invalidateUser(r.Name)
+
 	plog.Noticef("deleted a user: %s", r.Name)
 
 	return &pb.AuthUserDeleteResponse{}, nil
@@ -313,6 +343,11 @@ func (as *authStore) UserChangePassword(r *pb.AuthUserChangePasswordRequest) (*p
 
 	putUser(tx, updatedUser)
 
+	as.commitRevision(tx)
+
+	as.invalidateCachedPerm(r.Name)
+	as.invalidateUser(r.Name)
+
 	plog.Noticef("changed a password of a user: %s", r.Name)
 
 	return &pb.AuthUserChangePasswordResponse{}, nil
@@ -348,6 +383,8 @@ func (as *authStore) UserGrantRole(r *pb.AuthUserGrantRoleRequest) (*pb.AuthUser
 
 	as.invalidateCachedPerm(r.User)
 
+	as.commitRevision(tx)
+
 	plog.Noticef("granted role %s to user %s", r.Role, r.User)
 	return &pb.AuthUserGrantRoleResponse{}, nil
 }
@@ -416,6 +453,8 @@ func (as *authStore) UserRevokeRole(r *pb.AuthUserRevokeRoleRequest) (*pb.AuthUs
 
 	as.invalidateCachedPerm(r.Name)
 
+	as.commitRevision(tx)
+
 	plog.Noticef("revoked role %s from user %s", r.Role, r.Name)
 	return &pb.AuthUserRevokeRoleResponse{}, nil
 }
@@ -485,6 +524,8 @@ func (as *authStore) RoleRevokePermission(r *pb.AuthRoleRevokePermissionRequest)
 	// It should be optimized.
 	as.clearCachedPerm()
 
+	as.commitRevision(tx)
+
 	plog.Noticef("revoked key %s from role %s", r.Key, r.Role)
 	return &pb.AuthRoleRevokePermissionResponse{}, nil
 }
@@ -513,6 +554,8 @@ func (as *authStore) RoleDelete(r *pb.AuthRoleDeleteRequest) (*pb.AuthRoleDelete
 
 	delRole(tx, r.Role)
 
+	as.commitRevision(tx)
+
 	plog.Noticef("deleted role %s", r.Role)
 	return &pb.AuthRoleDeleteResponse{}, nil
 }
@@ -533,16 +576,18 @@ func (as *authStore) RoleAdd(r *pb.AuthRoleAddRequest) (*pb.AuthRoleAddResponse,
 
 	putRole(tx, newRole)
 
+	as.commitRevision(tx)
+
 	plog.Noticef("Role %s is created", r.Name)
 
 	return &pb.AuthRoleAddResponse{}, nil
 }
 
-func (as *authStore) UsernameFromToken(token string) (string, bool) {
+func (as *authStore) AuthInfoFromToken(token string) (*AuthInfo, bool) {
 	as.simpleTokensMu.RLock()
 	defer as.simpleTokensMu.RUnlock()
 	t, ok := as.simpleTokens[token]
-	return t, ok
+	return &AuthInfo{Username: t, Revision: as.revision}, ok
 }
 
 type permSlice []*authpb.Permission
@@ -594,15 +639,21 @@ func (as *authStore) RoleGrantPermission(r *pb.AuthRoleGrantPermissionRequest) (
 	// It should be optimized.
 	as.clearCachedPerm()
 
+	as.commitRevision(tx)
+
 	plog.Noticef("role %s's permission of key %s is updated as %s", r.Name, r.Perm.Key, authpb.Permission_Type_name[int32(r.Perm.PermType)])
 
 	return &pb.AuthRoleGrantPermissionResponse{}, nil
 }
 
-func (as *authStore) isOpPermitted(userName string, key, rangeEnd []byte, permTyp authpb.Permission_Type) bool {
+func (as *authStore) isOpPermitted(userName string, revision uint64, key, rangeEnd []byte, permTyp authpb.Permission_Type) error {
 	// TODO(mitake): this function would be costly so we need a caching mechanism
 	if !as.isAuthEnabled() {
-		return true
+		return nil
+	}
+
+	if revision < as.revision {
+		return ErrAuthOldRevision
 	}
 
 	tx := as.be.BatchTx()
@@ -612,43 +663,47 @@ func (as *authStore) isOpPermitted(userName string, key, rangeEnd []byte, permTy
 	user := getUser(tx, userName)
 	if user == nil {
 		plog.Errorf("invalid user name %s for permission checking", userName)
-		return false
+		return ErrPermissionDenied
 	}
 
 	if as.isRangeOpPermitted(tx, userName, key, rangeEnd, permTyp) {
-		return true
+		return nil
 	}
 
-	return false
+	return ErrPermissionDenied
 }
 
-func (as *authStore) IsPutPermitted(username string, key []byte) bool {
-	return as.isOpPermitted(username, key, nil, authpb.WRITE)
+func (as *authStore) IsPutPermitted(authInfo *AuthInfo, key []byte) error {
+	return as.isOpPermitted(authInfo.Username, authInfo.Revision, key, nil, authpb.WRITE)
 }
 
-func (as *authStore) IsRangePermitted(username string, key, rangeEnd []byte) bool {
-	return as.isOpPermitted(username, key, rangeEnd, authpb.READ)
+func (as *authStore) IsRangePermitted(authInfo *AuthInfo, key, rangeEnd []byte) error {
+	return as.isOpPermitted(authInfo.Username, authInfo.Revision, key, rangeEnd, authpb.READ)
 }
 
-func (as *authStore) IsDeleteRangePermitted(username string, key, rangeEnd []byte) bool {
-	return as.isOpPermitted(username, key, rangeEnd, authpb.WRITE)
+func (as *authStore) IsDeleteRangePermitted(authInfo *AuthInfo, key, rangeEnd []byte) error {
+	return as.isOpPermitted(authInfo.Username, authInfo.Revision, key, rangeEnd, authpb.WRITE)
 }
 
-func (as *authStore) IsAdminPermitted(username string) bool {
+func (as *authStore) IsAdminPermitted(authInfo *AuthInfo) error {
 	if !as.isAuthEnabled() {
-		return true
+		return nil
 	}
 
 	tx := as.be.BatchTx()
 	tx.Lock()
 	defer tx.Unlock()
 
-	u := getUser(tx, username)
+	u := getUser(tx, authInfo.Username)
 	if u == nil {
-		return false
+		return ErrUserNotFound
+	}
+
+	if !hasRootRole(u) {
+		return ErrPermissionDenied
 	}
 
-	return hasRootRole(u)
+	return nil
 }
 
 func getUser(tx backend.BatchTx, username string) *authpb.User {
@@ -760,13 +815,18 @@ func NewAuthStore(be backend.Backend) *authStore {
 	tx.UnsafeCreateBucket(authUsersBucketName)
 	tx.UnsafeCreateBucket(authRolesBucketName)
 
-	tx.Unlock()
-	be.ForceCommit()
-
-	return &authStore{
+	as := &authStore{
 		be:           be,
 		simpleTokens: make(map[string]string),
+		revision:     0,
 	}
+
+	as.commitRevision(tx)
+
+	tx.Unlock()
+	be.ForceCommit()
+
+	return as
 }
 
 func hasRootRole(u *authpb.User) bool {
@@ -777,3 +837,23 @@ func hasRootRole(u *authpb.User) bool {
 	}
 	return false
 }
+
+func (as *authStore) commitRevision(tx backend.BatchTx) {
+	as.revision++
+	revBytes := make([]byte, revBytesLen)
+	binary.BigEndian.PutUint64(revBytes, as.revision)
+	tx.UnsafePut(authBucketName, revisionKey, revBytes)
+}
+
+func getRevision(tx backend.BatchTx) uint64 {
+	_, vs := tx.UnsafeRange(authBucketName, []byte(revisionKey), nil, 0)
+	if len(vs) != 1 {
+		plog.Panicf("failed to get the key of auth store revision")
+	}
+
+	return binary.BigEndian.Uint64(vs[0])
+}
+
+func (as *authStore) Revision() uint64 {
+	return as.revision
+}

etcdserver/apply_auth.go

@@ -27,8 +27,9 @@ type authApplierV3 struct {
 
 	// mu serializes Apply so that user isn't corrupted and so that
 	// serialized requests don't leak data from TOCTOU errors
-	mu   sync.Mutex
-	user string
+	mu sync.Mutex
+
+	authInfo auth.AuthInfo
 }
 
 func newAuthApplierV3(as auth.AuthStore, base applierV3) *authApplierV3 {
@@ -41,46 +42,57 @@ func (aa *authApplierV3) Apply(r *pb.InternalRaftRequest) *applyResult {
 	if r.Header != nil {
 		// backward-compatible with pre-3.0 releases when internalRaftRequest
 		// does not have header field
-		aa.user = r.Header.Username
+		aa.authInfo.Username = r.Header.Username
+		aa.authInfo.Revision = r.Header.AuthRevision
 	}
-	if needAdminPermission(r) && !aa.as.IsAdminPermitted(aa.user) {
-		aa.user = ""
-		return &applyResult{err: auth.ErrPermissionDenied}
+	if needAdminPermission(r) {
+		if err := aa.as.IsAdminPermitted(&aa.authInfo); err != nil {
+			aa.authInfo.Username = ""
+			aa.authInfo.Revision = 0
+			return &applyResult{err: err}
+		}
 	}
 	ret := aa.applierV3.Apply(r)
-	aa.user = ""
+	aa.authInfo.Username = ""
+	aa.authInfo.Revision = 0
 	return ret
 }
 
 func (aa *authApplierV3) Put(txnID int64, r *pb.PutRequest) (*pb.PutResponse, error) {
-	if !aa.as.IsPutPermitted(aa.user, r.Key) {
-		return nil, auth.ErrPermissionDenied
+	if err := aa.as.IsPutPermitted(&aa.authInfo, r.Key); err != nil {
+		return nil, err
 	}
-	if r.PrevKv && !aa.as.IsRangePermitted(aa.user, r.Key, nil) {
-		return nil, auth.ErrPermissionDenied
+	if r.PrevKv {
+		err := aa.as.IsRangePermitted(&aa.authInfo, r.Key, nil)
+		if err != nil {
+			return nil, err
+		}
 	}
 	return aa.applierV3.Put(txnID, r)
 }
 
 func (aa *authApplierV3) Range(txnID int64, r *pb.RangeRequest) (*pb.RangeResponse, error) {
-	if !aa.as.IsRangePermitted(aa.user, r.Key, r.RangeEnd) {
-		return nil, auth.ErrPermissionDenied
+	if err := aa.as.IsRangePermitted(&aa.authInfo, r.Key, r.RangeEnd); err != nil {
+		return nil, err
 	}
 	return aa.applierV3.Range(txnID, r)
 }
 
 func (aa *authApplierV3) DeleteRange(txnID int64, r *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) {
-	if !aa.as.IsDeleteRangePermitted(aa.user, r.Key, r.RangeEnd) {
-		return nil, auth.ErrPermissionDenied
+	if err := aa.as.IsDeleteRangePermitted(&aa.authInfo, r.Key, r.RangeEnd); err != nil {
+		return nil, err
 	}
-	if r.PrevKv && !aa.as.IsRangePermitted(aa.user, r.Key, r.RangeEnd) {
-		return nil, auth.ErrPermissionDenied
+	if r.PrevKv {
+		err := aa.as.IsRangePermitted(&aa.authInfo, r.Key, r.RangeEnd)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	return aa.applierV3.DeleteRange(txnID, r)
 }
 
-func (aa *authApplierV3) checkTxnReqsPermission(reqs []*pb.RequestOp) bool {
+func (aa *authApplierV3) checkTxnReqsPermission(reqs []*pb.RequestOp) error {
 	for _, requ := range reqs {
 		switch tv := requ.Request.(type) {
 		case *pb.RequestOp_RequestRange:
@@ -88,8 +100,8 @@ func (aa *authApplierV3) checkTxnReqsPermission(reqs []*pb.RequestOp) bool {
 				continue
 			}
 
-			if !aa.as.IsRangePermitted(aa.user, tv.RequestRange.Key, tv.RequestRange.RangeEnd) {
-				return false
+			if err := aa.as.IsRangePermitted(&aa.authInfo, tv.RequestRange.Key, tv.RequestRange.RangeEnd); err != nil {
+				return err
 			}
 
 		case *pb.RequestOp_RequestPut:
@@ -97,8 +109,8 @@ func (aa *authApplierV3) checkTxnReqsPermission(reqs []*pb.RequestOp) bool {
 				continue
 			}
 
-			if !aa.as.IsPutPermitted(aa.user, tv.RequestPut.Key) {
-				return false
+			if err := aa.as.IsPutPermitted(&aa.authInfo, tv.RequestPut.Key); err != nil {
+				return err
 			}
 
 		case *pb.RequestOp_RequestDeleteRange:
@@ -106,31 +118,35 @@ func (aa *authApplierV3) checkTxnReqsPermission(reqs []*pb.RequestOp) bool {
 				continue
 			}
 
-			if tv.RequestDeleteRange.PrevKv && !aa.as.IsRangePermitted(aa.user, tv.RequestDeleteRange.Key, tv.RequestDeleteRange.RangeEnd) {
-				return false
+			if tv.RequestDeleteRange.PrevKv {
+				err := aa.as.IsRangePermitted(&aa.authInfo, tv.RequestDeleteRange.Key, tv.RequestDeleteRange.RangeEnd)
+				if err != nil {
+					return err
+				}
 			}
 
-			if !aa.as.IsDeleteRangePermitted(aa.user, tv.RequestDeleteRange.Key, tv.RequestDeleteRange.RangeEnd) {
-				return false
+			err := aa.as.IsDeleteRangePermitted(&aa.authInfo, tv.RequestDeleteRange.Key, tv.RequestDeleteRange.RangeEnd)
+			if err != nil {
+				return err
 			}
 		}
 	}
 
-	return true
+	return nil
 }
 
 func (aa *authApplierV3) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) {
 	for _, c := range rt.Compare {
-		if !aa.as.IsRangePermitted(aa.user, c.Key, nil) {
-			return nil, auth.ErrPermissionDenied
+		if err := aa.as.IsRangePermitted(&aa.authInfo, c.Key, nil); err != nil {
+			return nil, err
 		}
 	}
 
-	if !aa.checkTxnReqsPermission(rt.Success) {
-		return nil, auth.ErrPermissionDenied
+	if err := aa.checkTxnReqsPermission(rt.Success); err != nil {
+		return nil, err
 	}
-	if !aa.checkTxnReqsPermission(rt.Failure) {
-		return nil, auth.ErrPermissionDenied
+	if err := aa.checkTxnReqsPermission(rt.Failure); err != nil {
+		return nil, err
 	}
 
 	return aa.applierV3.Txn(rt)

etcdserver/etcdserverpb/raft_internal.pb.go

@@ -23,6 +23,8 @@ type RequestHeader struct {
 	ID uint64 `protobuf:"varint,1,opt,name=ID,json=iD,proto3" json:"ID,omitempty"`
 	// username is a username that is associated with an auth token of gRPC connection
 	Username string `protobuf:"bytes,2,opt,name=username,proto3" json:"username,omitempty"`
+	// auth_revision is a revision number of auth.authStore. It is not related to mvcc
+	AuthRevision uint64 `protobuf:"varint,3,opt,name=auth_revision,json=authRevision,proto3" json:"auth_revision,omitempty"`
 }
 
 func (m *RequestHeader) Reset()                    { *m = RequestHeader{} }
@@ -124,6 +126,11 @@ func (m *RequestHeader) MarshalTo(data []byte) (int, error) {
 		i = encodeVarintRaftInternal(data, i, uint64(len(m.Username)))
 		i += copy(data[i:], m.Username)
 	}
+	if m.AuthRevision != 0 {
+		data[i] = 0x18
+		i++
+		i = encodeVarintRaftInternal(data, i, uint64(m.AuthRevision))
+	}
 	return i, nil
 }
 
@@ -535,6 +542,9 @@ func (m *RequestHeader) Size() (n int) {
 	if l > 0 {
 		n += 1 + l + sovRaftInternal(uint64(l))
 	}
+	if m.AuthRevision != 0 {
+		n += 1 + sovRaftInternal(uint64(m.AuthRevision))
+	}
 	return n
 }
 
@@ -765,6 +775,25 @@ func (m *RequestHeader) Unmarshal(data []byte) error {
 			}
 			m.Username = string(data[iNdEx:postIndex])
 			iNdEx = postIndex
+		case 3:
+			if wireType != 0 {
+				return fmt.Errorf("proto: wrong wireType = %d for field AuthRevision", wireType)
+			}
+			m.AuthRevision = 0
+			for shift := uint(0); ; shift += 7 {
+				if shift >= 64 {
+					return ErrIntOverflowRaftInternal
+				}
+				if iNdEx >= l {
+					return io.ErrUnexpectedEOF
+				}
+				b := data[iNdEx]
+				iNdEx++
+				m.AuthRevision |= (uint64(b) & 0x7F) << shift
+				if b < 0x80 {
+					break
+				}
+			}
 		default:
 			iNdEx = preIndex
 			skippy, err := skipRaftInternal(data[iNdEx:])
@@ -2006,57 +2035,57 @@ var (
 )
 
 var fileDescriptorRaftInternal = []byte{
-	// 824 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x7c, 0x96, 0xdd, 0x4e, 0xdb, 0x48,
-	0x14, 0xc7, 0x49, 0xf8, 0xcc, 0x24, 0x64, 0xd9, 0x01, 0x76, 0x67, 0x83, 0x94, 0x85, 0xac, 0x76,
-	0x97, 0x7e, 0xd1, 0x0a, 0x2e, 0x7b, 0xd1, 0xa6, 0x04, 0x01, 0x12, 0x42, 0xc8, 0xa2, 0x52, 0xa5,
-	0x5e, 0x58, 0x43, 0x7c, 0x48, 0x5c, 0x1c, 0xdb, 0xb5, 0x27, 0x29, 0x7d, 0x93, 0x3e, 0x46, 0xbf,
-	0x1e, 0x82, 0x8b, 0x7e, 0xd0, 0xf6, 0x05, 0x5a, 0x7a, 0xd3, 0xab, 0xde, 0xb4, 0x0f, 0x50, 0xcd,
-	0x87, 0xc7, 0x71, 0x32, 0xe1, 0xce, 0x3e, 0xe7, 0x7f, 0x7e, 0xe7, 0x4c, 0xe6, 0x3f, 0x9e, 0xa0,
-	0xf9, 0x88, 0x1e, 0x33, 0xdb, 0xf5, 0x19, 0x44, 0x3e, 0xf5, 0xd6, 0xc2, 0x28, 0x60, 0x01, 0x2e,
-	0x01, 0x6b, 0x3a, 0x31, 0x44, 0x3d, 0x88, 0xc2, 0xa3, 0xca, 0x42, 0x2b, 0x68, 0x05, 0x22, 0x71,
-	0x93, 0x3f, 0x49, 0x4d, 0x65, 0x2e, 0xd5, 0xa8, 0x48, 0x21, 0x0a, 0x9b, 0xf2, 0xb1, 0x76, 0x1b,
-	0xcd, 0x5a, 0xf0, 0xb8, 0x0b, 0x31, 0xdb, 0x01, 0xea, 0x40, 0x84, 0xcb, 0x28, 0xbf, 0xdb, 0x20,
-	0xb9, 0xe5, 0xdc, 0xea, 0x84, 0x95, 0x77, 0x1b, 0xb8, 0x82, 0x66, 0xba, 0x31, 0x6f, 0xd9, 0x01,
-	0x92, 0x5f, 0xce, 0xad, 0x16, 0x2c, 0xfd, 0x5e, 0xfb, 0x5e, 0x46, 0xf3, 0xbb, 0x6a, 0x20, 0x8b,
-	0x1e, 0x33, 0x45, 0x1a, 0x62, 0xfc, 0x8b, 0xf2, 0xbd, 0x75, 0x51, 0x5d, 0x5c, 0x5f, 0x5c, 0xeb,
-	0x1f, 0x79, 0x4d, 0x95, 0x58, 0xf9, 0xde, 0x3a, 0xbe, 0x85, 0x26, 0x23, 0xea, 0xb7, 0x80, 0x8c,
-	0x0b, 0x65, 0x65, 0x40, 0xc9, 0x53, 0x89, 0x5c, 0x0a, 0xf1, 0x55, 0x34, 0x1e, 0x76, 0x19, 0x99,
-	0x10, 0x7a, 0x92, 0xd5, 0x1f, 0x74, 0x93, 0x79, 0x2c, 0x2e, 0xc2, 0x9b, 0xa8, 0xe4, 0x80, 0x07,
-	0x0c, 0x6c, 0xd9, 0x64, 0x52, 0x14, 0x2d, 0x67, 0x8b, 0x1a, 0x42, 0x91, 0x69, 0x55, 0x74, 0xd2,
-	0x18, 0x6f, 0xc8, 0x4e, 0x7d, 0x32, 0x65, 0x6a, 0x78, 0x78, 0xea, 0xeb, 0x86, 0xec, 0xd4, 0xc7,
-	0x77, 0x10, 0x6a, 0x06, 0x9d, 0x90, 0x36, 0x99, 0x1b, 0xf8, 0x64, 0x5a, 0x94, 0xfc, 0x9d, 0x2d,
-	0xd9, 0xd4, 0xf9, 0xa4, 0xb2, 0xaf, 0x04, 0xdf, 0x45, 0x45, 0x0f, 0x68, 0x0c, 0x76, 0x2b, 0xa2,
-	0x3e, 0x23, 0x33, 0x26, 0xc2, 0x1e, 0x17, 0x6c, 0xf3, 0xbc, 0x26, 0x78, 0x3a, 0xc4, 0xd7, 0x2c,
-	0x09, 0x11, 0xf4, 0x82, 0x13, 0x20, 0x05, 0xd3, 0x9a, 0x05, 0xc2, 0x12, 0x02, 0xbd, 0x66, 0x2f,
-	0x8d, 0xf1, 0x6d, 0xa1, 0x1e, 0x8d, 0x3a, 0x04, 0x99, 0xb6, 0xa5, 0xce, 0x53, 0x7a, 0x5b, 0x84,
-	0x10, 0x6f, 0xa0, 0xa9, 0xb6, 0x70, 0x13, 0x71, 0x44, 0xc9, 0x92, 0x71, 0xcf, 0xa5, 0xe1, 0x2c,
-	0x25, 0xc5, 0x75, 0x54, 0xa4, 0x5d, 0xd6, 0xb6, 0xc1, 0xa7, 0x47, 0x1e, 0x90, 0x6f, 0xc6, 0x1f,
-	0xac, 0xde, 0x65, 0xed, 0x2d, 0x21, 0xd0, 0xcb, 0xa5, 0x3a, 0x84, 0x1b, 0xa8, 0x24, 0x10, 0x8e,
-	0x1b, 0x0b, 0xc6, 0x8f, 0x69, 0xd3, 0x7a, 0x39, 0xa3, 0x21, 0x15, 0x7a, 0xbd, 0x34, 0x8d, 0xe1,
-	0x7d, 0x49, 0x01, 0x9f, 0xb9, 0x4d, 0xca, 0x80, 0xfc, 0x94, 0x94, 0x2b, 0x59, 0x4a, 0xe2, 0xfb,
-	0x7a, 0x9f, 0x34, 0xc1, 0x65, 0xea, 0xf1, 0x16, 0x9a, 0x15, 0x53, 0xf1, 0x63, 0x63, 0x53, 0xc7,
-	0x21, 0x6f, 0x66, 0x46, 0x8d, 0x75, 0x3f, 0x86, 0xa8, 0xee, 0x38, 0x99, 0xb1, 0x54, 0x0c, 0xef,
-	0xa3, 0xb9, 0x14, 0x23, 0x3d, 0x49, 0xde, 0x4a, 0xd2, 0x3f, 0x66, 0x92, 0x32, 0xb3, 0x82, 0x95,
-	0x69, 0x26, 0x9c, 0x1d, 0xab, 0x05, 0x8c, 0xbc, 0xbb, 0x74, 0xac, 0x6d, 0x60, 0x43, 0x63, 0x6d,
-	0x03, 0xc3, 0x2d, 0xf4, 0x57, 0x8a, 0x69, 0xb6, 0xf9, 0x29, 0xb1, 0x43, 0x1a, 0xc7, 0x4f, 0x82,
-	0xc8, 0x21, 0xef, 0x25, 0xf2, 0x9a, 0x19, 0xb9, 0x29, 0xd4, 0x07, 0x4a, 0x9c, 0xd0, 0xff, 0xa0,
-	0xc6, 0x34, 0x7e, 0x80, 0x16, 0xfa, 0xe6, 0xe5, 0xf6, 0xb6, 0xa3, 0xc0, 0x03, 0x72, 0x2e, 0x7b,
-	0xfc, 0x37, 0x62, 0x6c, 0x71, 0x34, 0x82, 0x74, 0xab, 0x7f, 0xa7, 0x83, 0x19, 0xfc, 0x10, 0x2d,
-	0xa6, 0x64, 0x79, 0x52, 0x24, 0xfa, 0x83, 0x44, 0xff, 0x6f, 0x46, 0xab, 0x23, 0xd3, 0xc7, 0xc6,
-	0x74, 0x28, 0x85, 0x77, 0x50, 0x39, 0x85, 0x7b, 0x6e, 0xcc, 0xc8, 0x47, 0x49, 0x5d, 0x31, 0x53,
-	0xf7, 0xdc, 0x98, 0x65, 0x7c, 0x94, 0x04, 0x35, 0x89, 0x8f, 0x26, 0x49, 0x9f, 0x46, 0x92, 0x78,
-	0xeb, 0x21, 0x52, 0x12, 0xd4, 0x5b, 0x2f, 0x48, 0xdc, 0x91, 0xcf, 0x0b, 0xa3, 0xb6, 0x9e, 0xd7,
-	0x0c, 0x3a, 0x52, 0xc5, 0xb4, 0x23, 0x05, 0x46, 0x39, 0xf2, 0x45, 0x61, 0x94, 0x23, 0x79, 0x95,
-	0xc1, 0x91, 0x69, 0x38, 0x3b, 0x16, 0x77, 0xe4, 0xcb, 0x4b, 0xc7, 0x1a, 0x74, 0xa4, 0x8a, 0xe1,
-	0x47, 0xa8, 0xd2, 0x87, 0x11, 0x46, 0x09, 0x21, 0xea, 0xb8, 0x71, 0xcc, 0xbf, 0xc3, 0xaf, 0x24,
-	0xf3, 0xfa, 0x08, 0x26, 0x97, 0x1f, 0x68, 0x75, 0xc2, 0xff, 0x93, 0x9a, 0xf3, 0xb8, 0x83, 0x96,
-	0xd2, 0x5e, 0xca, 0x3a, 0x7d, 0xcd, 0x5e, 0xcb, 0x66, 0x37, 0xcc, 0xcd, 0xa4, 0x4b, 0x86, 0xbb,
-	0x11, 0x3a, 0x42, 0x50, 0xfb, 0x0d, 0xcd, 0x6e, 0x75, 0x42, 0xf6, 0xd4, 0x82, 0x38, 0x0c, 0xfc,
-	0x18, 0x6a, 0x21, 0x5a, 0xba, 0xe4, 0x43, 0x84, 0x31, 0x9a, 0x10, 0x17, 0x77, 0x4e, 0x5c, 0xdc,
-	0xe2, 0x99, 0x5f, 0xe8, 0xfa, 0x7c, 0xaa, 0x0b, 0x3d, 0x79, 0xc7, 0x2b, 0xa8, 0x14, 0xbb, 0x9d,
-	0xd0, 0x03, 0x9b, 0x05, 0x27, 0xe0, 0x8b, 0x8b, 0xb8, 0x60, 0x15, 0x65, 0xec, 0x90, 0x87, 0xee,
-	0x2d, 0x9c, 0x7d, 0xa9, 0x8e, 0x9d, 0x5d, 0x54, 0x73, 0xe7, 0x17, 0xd5, 0xdc, 0xe7, 0x8b, 0x6a,
-	0xee, 0xd9, 0xd7, 0xea, 0xd8, 0xd1, 0x94, 0xf8, 0x37, 0xb1, 0xf1, 0x2b, 0x00, 0x00, 0xff, 0xff,
-	0x54, 0x8c, 0x4a, 0x7f, 0xa5, 0x08, 0x00, 0x00,
+	// 831 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x7c, 0x56, 0xdb, 0x4e, 0xdb, 0x4a,
+	0x14, 0x25, 0xe1, 0x9a, 0x49, 0xc8, 0xe1, 0x0c, 0x70, 0xce, 0x9c, 0x20, 0x71, 0x20, 0xa8, 0xf7,
+	0x96, 0x56, 0xf0, 0x01, 0x6d, 0x4a, 0x10, 0x20, 0xa1, 0x0a, 0x59, 0x54, 0xaa, 0xd4, 0x07, 0x77,
+	0x88, 0xa7, 0xc1, 0xad, 0x63, 0xbb, 0xf6, 0x84, 0xd2, 0x3f, 0xe9, 0x67, 0xf4, 0xf6, 0x11, 0x3c,
+	0xf4, 0x42, 0xdb, 0x1f, 0xe8, 0xe5, 0xa5, 0x4f, 0x7d, 0x69, 0x3f, 0xa0, 0x73, 0xf3, 0xd8, 0x4e,
+	0xc6, 0x3c, 0x44, 0xb2, 0xd7, 0xac, 0xbd, 0xd6, 0x1e, 0xef, 0x35, 0x0c, 0x60, 0x36, 0xc2, 0x0f,
+	0xa9, 0xed, 0xfa, 0x94, 0x44, 0x3e, 0xf6, 0x56, 0xc3, 0x28, 0xa0, 0x01, 0xac, 0x11, 0xda, 0x71,
+	0x62, 0x12, 0x1d, 0x91, 0x28, 0x3c, 0x68, 0xcc, 0x75, 0x83, 0x6e, 0x20, 0x16, 0xae, 0xf3, 0x27,
+	0xc9, 0x69, 0xcc, 0xa4, 0x1c, 0x85, 0x54, 0xa2, 0xb0, 0x23, 0x1f, 0x9b, 0x0f, 0xc0, 0xb4, 0x45,
+	0x9e, 0xf4, 0x49, 0x4c, 0xb7, 0x09, 0x76, 0x48, 0x04, 0xeb, 0xa0, 0xbc, 0xd3, 0x46, 0xa5, 0xa5,
+	0xd2, 0xc5, 0x31, 0xab, 0xec, 0xb6, 0x61, 0x03, 0x4c, 0xf5, 0x63, 0x6e, 0xd9, 0x23, 0xa8, 0xcc,
+	0xd0, 0x8a, 0xa5, 0xdf, 0xe1, 0x0a, 0x98, 0xc6, 0x7d, 0x7a, 0x68, 0x47, 0xe4, 0xc8, 0x8d, 0xdd,
+	0xc0, 0x47, 0xa3, 0xa2, 0xac, 0xc6, 0x41, 0x4b, 0x61, 0xcd, 0x9f, 0x75, 0x30, 0xbb, 0xa3, 0xba,
+	0xb6, 0xd8, 0x16, 0x94, 0xdd, 0x90, 0xd1, 0x39, 0x50, 0x3e, 0x5a, 0x13, 0x16, 0xd5, 0xb5, 0xf9,
+	0xd5, 0xec, 0xbe, 0x56, 0x55, 0x89, 0xc5, 0x08, 0xf0, 0x06, 0x18, 0x8f, 0xb0, 0xdf, 0x25, 0xc2,
+	0xab, 0xba, 0xd6, 0x18, 0x60, 0xf2, 0xa5, 0x84, 0x2e, 0x89, 0xf0, 0x32, 0x18, 0x0d, 0xfb, 0x14,
+	0x8d, 0x09, 0x3e, 0xca, 0xf3, 0xf7, 0xfa, 0x49, 0x3f, 0x16, 0x27, 0xc1, 0x0d, 0x50, 0x73, 0x88,
+	0x47, 0x28, 0xb1, 0xa5, 0xc9, 0xb8, 0x28, 0x5a, 0xca, 0x17, 0xb5, 0x05, 0x23, 0x67, 0x55, 0x75,
+	0x52, 0x8c, 0x1b, 0xd2, 0x63, 0x1f, 0x4d, 0x98, 0x0c, 0xf7, 0x8f, 0x7d, 0x6d, 0xc8, 0x48, 0xf0,
+	0x26, 0x00, 0x9d, 0xa0, 0x17, 0xe2, 0x0e, 0xe5, 0xdf, 0x6f, 0x52, 0x94, 0xfc, 0x9f, 0x2f, 0xd9,
+	0xd0, 0xeb, 0x49, 0x65, 0xa6, 0x04, 0xde, 0x02, 0x55, 0x8f, 0xe0, 0x98, 0xd8, 0x5d, 0xd6, 0x31,
+	0x45, 0x53, 0x26, 0x85, 0x5d, 0x4e, 0xd8, 0xe2, 0xeb, 0x5a, 0xc1, 0xd3, 0x10, 0xdf, 0xb3, 0x54,
+	0x60, 0x63, 0x0c, 0x1e, 0x13, 0x54, 0x31, 0xed, 0x59, 0x48, 0x58, 0x82, 0xa0, 0xf7, 0xec, 0xa5,
+	0x18, 0x1f, 0x0b, 0xf6, 0x70, 0xd4, 0x43, 0xc0, 0x34, 0x96, 0x16, 0x5f, 0xd2, 0x63, 0x11, 0x44,
+	0xb8, 0x0e, 0x26, 0x0e, 0x45, 0xe4, 0x90, 0x23, 0x4a, 0x16, 0x8c, 0x33, 0x97, 0xa9, 0xb4, 0x14,
+	0x15, 0xb6, 0x40, 0x55, 0x24, 0x8e, 0xf8, 0xf8, 0xc0, 0x23, 0xe8, 0x87, 0xf1, 0x83, 0xb5, 0x18,
+	0x63, 0x53, 0x10, 0xf4, 0x76, 0xb1, 0x86, 0x60, 0x1b, 0x88, 0x7c, 0xda, 0x8e, 0x1b, 0x0b, 0x8d,
+	0x5f, 0x93, 0xa6, 0xfd, 0x72, 0x8d, 0xb6, 0x64, 0xe8, 0xfd, 0xe2, 0x14, 0x83, 0x77, 0xa4, 0x0a,
+	0xf1, 0xa9, 0xdb, 0xc1, 0x94, 0xa0, 0xdf, 0x52, 0xe5, 0x52, 0x5e, 0x25, 0xc9, 0x7d, 0x2b, 0x43,
+	0x4d, 0xe4, 0x72, 0xf5, 0x70, 0x53, 0x1d, 0x25, 0x7e, 0xb6, 0x6c, 0xec, 0x38, 0xe8, 0xed, 0x54,
+	0x51, 0x5b, 0x77, 0xd9, 0x5b, 0xcb, 0x71, 0x72, 0x6d, 0x29, 0x8c, 0xb5, 0x35, 0x93, 0xca, 0xc8,
+	0x4c, 0xa2, 0x77, 0x52, 0x69, 0xc5, 0xac, 0xa4, 0xc2, 0xac, 0xc4, 0xea, 0x38, 0x07, 0xe7, 0xdb,
+	0xea, 0x12, 0x8a, 0xde, 0x9f, 0xd9, 0xd6, 0x16, 0xa1, 0x43, 0x6d, 0x31, 0x0c, 0x76, 0xc1, 0x7f,
+	0xa9, 0x4c, 0xe7, 0x90, 0x9f, 0x12, 0x3b, 0xc4, 0x71, 0xfc, 0x34, 0x88, 0x1c, 0xf4, 0x41, 0x4a,
+	0x5e, 0x31, 0x4b, 0x6e, 0x08, 0xf6, 0x9e, 0x22, 0x27, 0xea, 0xff, 0x60, 0xe3, 0x32, 0xbc, 0x07,
+	0xe6, 0x32, 0xfd, 0xf2, 0x78, 0xdb, 0x51, 0xc0, 0x86, 0x7c, 0x2a, 0x3d, 0xce, 0x17, 0xb4, 0x2d,
+	0x8e, 0x46, 0x90, 0x8e, 0xfa, 0x6f, 0x3c, 0xb8, 0x02, 0xef, 0x83, 0xf9, 0x54, 0x59, 0x9e, 0x14,
+	0x29, 0xfd, 0x51, 0x4a, 0x5f, 0x30, 0x4b, 0xab, 0x23, 0x93, 0xd1, 0x86, 0x78, 0x68, 0x09, 0x6e,
+	0x83, 0x7a, 0x2a, 0xee, 0xb9, 0x31, 0x45, 0x9f, 0xa4, 0xea, 0xb2, 0x59, 0x75, 0x97, 0x51, 0x72,
+	0x39, 0x4a, 0x40, 0xad, 0xc4, 0x5b, 0x93, 0x4a, 0x9f, 0x0b, 0x95, 0xb8, 0xf5, 0x90, 0x52, 0x02,
+	0xea, 0xd1, 0x0b, 0x25, 0x9e, 0xc8, 0x17, 0x95, 0xa2, 0xd1, 0xf3, 0x9a, 0xc1, 0x44, 0x2a, 0x4c,
+	0x27, 0x52, 0xc8, 0xa8, 0x44, 0xbe, 0xac, 0x14, 0x25, 0x92, 0x57, 0x19, 0x12, 0x99, 0xc2, 0xf9,
+	0xb6, 0x78, 0x22, 0x5f, 0x9d, 0xd9, 0xd6, 0x60, 0x22, 0x15, 0x06, 0x1f, 0x81, 0x46, 0x46, 0x46,
+	0x04, 0x25, 0x24, 0x51, 0xcf, 0x8d, 0xc5, 0x3d, 0xf6, 0x5a, 0x6a, 0x5e, 0x2d, 0xd0, 0xe4, 0xf4,
+	0x3d, 0xcd, 0x4e, 0xf4, 0xff, 0xc5, 0xe6, 0x75, 0xd8, 0x03, 0x0b, 0xa9, 0x97, 0x8a, 0x4e, 0xc6,
+	0xec, 0x8d, 0x34, 0xbb, 0x66, 0x36, 0x93, 0x29, 0x19, 0x76, 0x43, 0xb8, 0x80, 0xd0, 0xfc, 0x0b,
+	0x4c, 0x6f, 0xf6, 0x42, 0xfa, 0xcc, 0x22, 0x71, 0x18, 0xf8, 0x31, 0x69, 0x86, 0x60, 0xe1, 0x8c,
+	0x3f, 0x44, 0x10, 0x82, 0x31, 0x71, 0xbb, 0x97, 0xc4, 0xed, 0x2e, 0x9e, 0xf9, 0xad, 0xaf, 0xcf,
+	0xa7, 0xba, 0xf5, 0x93, 0x77, 0xb8, 0x0c, 0x6a, 0xb1, 0xdb, 0x0b, 0xd9, 0x5e, 0x28, 0x33, 0x96,
+	0x97, 0x7e, 0xc5, 0xaa, 0x4a, 0x6c, 0x9f, 0x43, 0xb7, 0xe7, 0x4e, 0xbe, 0x2e, 0x8e, 0x9c, 0x7c,
+	0x5b, 0x2c, 0x9d, 0xb2, 0xdf, 0x17, 0xf6, 0x7b, 0xfe, 0x7d, 0x71, 0xe4, 0x60, 0x42, 0xfc, 0xcb,
+	0xb1, 0xfe, 0x27, 0x00, 0x00, 0xff, 0xff, 0x02, 0x23, 0xd2, 0x00, 0xca, 0x08, 0x00, 0x00,
 }

etcdserver/etcdserverpb/raft_internal.proto

@@ -14,6 +14,8 @@ message RequestHeader {
   uint64 ID = 1;
   // username is a username that is associated with an auth token of gRPC connection
   string username = 2;
+  // auth_revision is a revision number of auth.authStore. It is not related to mvcc
+  uint64 auth_revision = 3;
 }
 
 // An InternalRaftRequest is the union of all requests which can be

etcdserver/v3_server.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/coreos/etcd/auth"
 	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
 	"github.com/coreos/etcd/lease"
 	"github.com/coreos/etcd/lease/leasehttp"
@@ -87,15 +88,34 @@ func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeRe
 	var err error
 
 	if r.Serializable {
-		var user string
-		user, err = s.usernameFromCtx(ctx)
-		if err != nil {
-			return nil, err
+		for {
+			authInfo, err := s.authInfoFromCtx(ctx)
+			if err != nil {
+				return nil, err
+			}
+
+			hdr := &pb.RequestHeader{}
+			if authInfo != nil {
+				hdr.Username = authInfo.Username
+				hdr.AuthRevision = authInfo.Revision
+			}
+
+			result = s.applyV3.Apply(&pb.InternalRaftRequest{Header: hdr, Range: r})
+
+			if result.err != nil {
+				if result.err == auth.ErrAuthOldRevision {
+					continue
+				}
+				break
+			}
+
+			if authInfo == nil || authInfo.Revision == s.authStore.Revision() {
+				break
+			}
+
+			// The revision that authorized this request is obsolete.
+			// For avoiding TOCTOU problem, retry of the request is required.
 		}
-		result = s.applyV3.Apply(
-			&pb.InternalRaftRequest{
-				Header: &pb.RequestHeader{Username: user},
-				Range:  r})
 	} else {
 		result, err = s.processInternalRaftRequest(ctx, pb.InternalRaftRequest{Range: r})
 	}
@@ -135,14 +155,34 @@ func (s *EtcdServer) Txn(ctx context.Context, r *pb.TxnRequest) (*pb.TxnResponse
 	var err error
 
 	if isTxnSerializable(r) {
-		user, err := s.usernameFromCtx(ctx)
-		if err != nil {
-			return nil, err
+		for {
+			authInfo, err := s.authInfoFromCtx(ctx)
+			if err != nil {
+				return nil, err
+			}
+
+			hdr := &pb.RequestHeader{}
+			if authInfo != nil {
+				hdr.Username = authInfo.Username
+				hdr.AuthRevision = authInfo.Revision
+			}
+
+			result = s.applyV3.Apply(&pb.InternalRaftRequest{Header: hdr, Txn: r})
+
+			if result.err != nil {
+				if result.err == auth.ErrAuthOldRevision {
+					continue
+				}
+				break
+			}
+
+			if authInfo == nil || authInfo.Revision == s.authStore.Revision() {
+				break
+			}
+
+			// The revision that authorized this request is obsolete.
+			// For avoiding TOCTOU problem, retry of this request is required.
 		}
-		result = s.applyV3.Apply(
-			&pb.InternalRaftRequest{
-				Header: &pb.RequestHeader{Username: user},
-				Txn:    r})
 	} else {
 		result, err = s.processInternalRaftRequest(ctx, pb.InternalRaftRequest{Txn: r})
 	}
@@ -170,7 +210,7 @@ func isTxnSerializable(r *pb.TxnRequest) bool {
 }
 
 func (s *EtcdServer) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) {
-	result, err := s.processInternalRaftRequest(ctx, pb.InternalRaftRequest{Compaction: r})
+	result, err := s.processInternalRaftRequestOnce(ctx, pb.InternalRaftRequest{Compaction: r})
 	if r.Physical && result != nil && result.physc != nil {
 		<-result.physc
 		// The compaction is done deleting keys; the hash is now settled
@@ -203,7 +243,7 @@ func (s *EtcdServer) LeaseGrant(ctx context.Context, r *pb.LeaseGrantRequest) (*
 		// only use positive int64 id's
 		r.ID = int64(s.reqIDGen.Next() & ((1 << 63) - 1))
 	}
-	result, err := s.processInternalRaftRequest(ctx, pb.InternalRaftRequest{LeaseGrant: r})
+	result, err := s.processInternalRaftRequestOnce(ctx, pb.InternalRaftRequest{LeaseGrant: r})
 	if err != nil {
 		return nil, err
 	}
@@ -214,7 +254,7 @@ func (s *EtcdServer) LeaseGrant(ctx context.Context, r *pb.LeaseGrantRequest) (*
 }
 
 func (s *EtcdServer) LeaseRevoke(ctx context.Context, r *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error) {
-	result, err := s.processInternalRaftRequest(ctx, pb.InternalRaftRequest{LeaseRevoke: r})
+	result, err := s.processInternalRaftRequestOnce(ctx, pb.InternalRaftRequest{LeaseRevoke: r})
 	if err != nil {
 		return nil, err
 	}
@@ -260,7 +300,7 @@ func (s *EtcdServer) LeaseRenew(id lease.LeaseID) (int64, error) {
 }
 
 func (s *EtcdServer) Alarm(ctx context.Context, r *pb.AlarmRequest) (*pb.AlarmResponse, error) {
-	result, err := s.processInternalRaftRequest(ctx, pb.InternalRaftRequest{Alarm: r})
+	result, err := s.processInternalRaftRequestOnce(ctx, pb.InternalRaftRequest{Alarm: r})
 	if err != nil {
 		return nil, err
 	}
@@ -271,7 +311,7 @@ func (s *EtcdServer) Alarm(ctx context.Context, r *pb.AlarmRequest) (*pb.AlarmRe
 }
 
 func (s *EtcdServer) AuthEnable(ctx context.Context, r *pb.AuthEnableRequest) (*pb.AuthEnableResponse, error) {
-	result, err := s.processInternalRaftRequest(ctx, pb.InternalRaftRequest{AuthEnable: r})
+	result, err := s.processInternalRaftRequestOnce(ctx, pb.InternalRaftRequest{AuthEnable: r})
 	if err != nil {
 		return nil, err
 	}
@@ -304,7 +344,7 @@ func (s *EtcdServer) Authenticate(ctx context.Context, r *pb.AuthenticateRequest
 		SimpleToken: st,
 	}
 
-	result, err := s.processInternalRaftRequest(ctx, pb.InternalRaftRequest{Authenticate: internalReq})
+	result, err := s.processInternalRaftRequestOnce(ctx, pb.InternalRaftRequest{Authenticate: internalReq})
 	if err != nil {
 		return nil, err
 	}
@@ -487,31 +527,31 @@ func (s *EtcdServer) isValidSimpleToken(token string) bool {
 	return true
 }
 
-func (s *EtcdServer) usernameFromCtx(ctx context.Context) (string, error) {
+func (s *EtcdServer) authInfoFromCtx(ctx context.Context) (*auth.AuthInfo, error) {
 	md, ok := metadata.FromContext(ctx)
 	if !ok {
-		return "", nil
+		return nil, nil
 	}
 
 	ts, tok := md["token"]
 	if !tok {
-		return "", nil
+		return nil, nil
 	}
 
 	token := ts[0]
 	if !s.isValidSimpleToken(token) {
-		return "", ErrInvalidAuthToken
+		return nil, ErrInvalidAuthToken
 	}
 
-	username, uok := s.AuthStore().UsernameFromToken(token)
+	authInfo, uok := s.AuthStore().AuthInfoFromToken(token)
 	if !uok {
 		plog.Warningf("invalid auth token: %s", token)
-		return "", ErrInvalidAuthToken
+		return nil, ErrInvalidAuthToken
 	}
-	return username, nil
+	return authInfo, nil
 }
 
-func (s *EtcdServer) processInternalRaftRequest(ctx context.Context, r pb.InternalRaftRequest) (*applyResult, error) {
+func (s *EtcdServer) processInternalRaftRequestOnce(ctx context.Context, r pb.InternalRaftRequest) (*applyResult, error) {
 	ai := s.getAppliedIndex()
 	ci := s.getCommittedIndex()
 	if ci > ai+maxGapBetweenApplyAndCommitIndex {
@@ -521,11 +561,15 @@ func (s *EtcdServer) processInternalRaftRequest(ctx context.Context, r pb.Intern
 	r.Header = &pb.RequestHeader{
 		ID: s.reqIDGen.Next(),
 	}
-	username, err := s.usernameFromCtx(ctx)
+
+	authInfo, err := s.authInfoFromCtx(ctx)
 	if err != nil {
 		return nil, err
 	}
-	r.Header.Username = username
+	if authInfo != nil {
+		r.Header.Username = authInfo.Username
+		r.Header.AuthRevision = authInfo.Revision
+	}
 
 	data, err := r.Marshal()
 	if err != nil {
@@ -562,5 +606,18 @@ func (s *EtcdServer) processInternalRaftRequest(ctx context.Context, r pb.Intern
 	}
 }
 
+func (s *EtcdServer) processInternalRaftRequest(ctx context.Context, r pb.InternalRaftRequest) (*applyResult, error) {
+	var result *applyResult
+	var err error
+	for {
+		result, err = s.processInternalRaftRequestOnce(ctx, r)
+		if err != auth.ErrAuthOldRevision {
+			break
+		}
+	}
+
+	return result, err
+}
+
 // Watchable returns a watchable interface attached to the etcdserver.
 func (s *EtcdServer) Watchable() mvcc.WatchableKV { return s.KV() }