etcd-io__etcd/functional/rpcpb/member.go

// Copyright 2018 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rpcpb

import (
	"context"
	"crypto/tls"
	"fmt"
	"net/url"
	"os"
	"time"

	"go.etcd.io/etcd/clientv3"
	"go.etcd.io/etcd/clientv3/snapshot"
	pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
	"go.etcd.io/etcd/pkg/logutil"
	"go.etcd.io/etcd/pkg/transport"

	"github.com/dustin/go-humanize"
	"go.uber.org/zap"
	grpc "google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
)

// ElectionTimeout returns an election timeout duration.
func (m *Member) ElectionTimeout() time.Duration {
	return time.Duration(m.Etcd.ElectionTimeoutMs) * time.Millisecond
}

// DialEtcdGRPCServer creates a raw gRPC connection to an etcd member.
func (m *Member) DialEtcdGRPCServer(opts ...grpc.DialOption) (*grpc.ClientConn, error) {
	dialOpts := []grpc.DialOption{
		grpc.WithTimeout(5 * time.Second),
		grpc.WithBlock(),
	}

	secure := false
	for _, cu := range m.Etcd.AdvertiseClientURLs {
		u, err := url.Parse(cu)
		if err != nil {
			return nil, err
		}
		if u.Scheme == "https" { // TODO: handle unix
			secure = true
		}
	}

	if secure {
		// assume save TLS assets are already stord on disk
		tlsInfo := transport.TLSInfo{
			CertFile:      m.ClientCertPath,
			KeyFile:       m.ClientKeyPath,
			TrustedCAFile: m.ClientTrustedCAPath,

			// TODO: remove this with generated certs
			// only need it for auto TLS
			InsecureSkipVerify: true,
		}
		tlsConfig, err := tlsInfo.ClientConfig()
		if err != nil {
			return nil, err
		}
		creds := credentials.NewTLS(tlsConfig)
		dialOpts = append(dialOpts, grpc.WithTransportCredentials(creds))
	} else {
		dialOpts = append(dialOpts, grpc.WithInsecure())
	}
	dialOpts = append(dialOpts, opts...)
	return grpc.Dial(m.EtcdClientEndpoint, dialOpts...)
}

// CreateEtcdClientConfig creates a client configuration from member.
func (m *Member) CreateEtcdClientConfig(opts ...grpc.DialOption) (cfg *clientv3.Config, err error) {
	secure := false
	for _, cu := range m.Etcd.AdvertiseClientURLs {
		var u *url.URL
		u, err = url.Parse(cu)
		if err != nil {
			return nil, err
		}
		if u.Scheme == "https" { // TODO: handle unix
			secure = true
		}
	}

	// TODO: make this configurable
	level := "error"
	if os.Getenv("ETCD_CLIENT_DEBUG") != "" {
		level = "debug"
	}
	lcfg := logutil.DefaultZapLoggerConfig
	lcfg.Level = zap.NewAtomicLevelAt(logutil.ConvertToZapLevel(level))

	cfg = &clientv3.Config{
		Endpoints:   []string{m.EtcdClientEndpoint},
		DialTimeout: 10 * time.Second,
		DialOptions: opts,
		LogConfig:   &lcfg,
	}
	if secure {
		// assume save TLS assets are already stord on disk
		tlsInfo := transport.TLSInfo{
			CertFile:      m.ClientCertPath,
			KeyFile:       m.ClientKeyPath,
			TrustedCAFile: m.ClientTrustedCAPath,

			// TODO: remove this with generated certs
			// only need it for auto TLS
			InsecureSkipVerify: true,
		}
		var tlsConfig *tls.Config
		tlsConfig, err = tlsInfo.ClientConfig()
		if err != nil {
			return nil, err
		}
		cfg.TLS = tlsConfig
	}
	return cfg, err
}

// CreateEtcdClient creates a client from member.
func (m *Member) CreateEtcdClient(opts ...grpc.DialOption) (*clientv3.Client, error) {
	cfg, err := m.CreateEtcdClientConfig(opts...)
	if err != nil {
		return nil, err
	}
	return clientv3.New(*cfg)
}

// CheckCompact ensures that historical data before given revision has been compacted.
func (m *Member) CheckCompact(rev int64) error {
	cli, err := m.CreateEtcdClient()
	if err != nil {
		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	defer cli.Close()

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	wch := cli.Watch(ctx, "\x00", clientv3.WithFromKey(), clientv3.WithRev(rev-1))
	wr, ok := <-wch
	cancel()

	if !ok {
		return fmt.Errorf("watch channel terminated (endpoint %q)", m.EtcdClientEndpoint)
	}
	if wr.CompactRevision != rev {
		return fmt.Errorf("got compact revision %v, wanted %v (endpoint %q)", wr.CompactRevision, rev, m.EtcdClientEndpoint)
	}

	return nil
}

// Defrag runs defragmentation on this member.
func (m *Member) Defrag() error {
	cli, err := m.CreateEtcdClient()
	if err != nil {
		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	defer cli.Close()

	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
	_, err = cli.Defragment(ctx, m.EtcdClientEndpoint)
	cancel()
	return err
}

// RevHash fetches current revision and hash on this member.
func (m *Member) RevHash() (int64, int64, error) {
	conn, err := m.DialEtcdGRPCServer()
	if err != nil {
		return 0, 0, err
	}
	defer conn.Close()

	mt := pb.NewMaintenanceClient(conn)
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	resp, err := mt.Hash(ctx, &pb.HashRequest{}, grpc.FailFast(false))
	cancel()

	if err != nil {
		return 0, 0, err
	}

	return resp.Header.Revision, int64(resp.Hash), nil
}

// Rev fetches current revision on this member.
func (m *Member) Rev(ctx context.Context) (int64, error) {
	cli, err := m.CreateEtcdClient()
	if err != nil {
		return 0, fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	defer cli.Close()

	resp, err := cli.Status(ctx, m.EtcdClientEndpoint)
	if err != nil {
		return 0, err
	}
	return resp.Header.Revision, nil
}

// Compact compacts member storage with given revision.
// It blocks until it's physically done.
func (m *Member) Compact(rev int64, timeout time.Duration) error {
	cli, err := m.CreateEtcdClient()
	if err != nil {
		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	defer cli.Close()

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	_, err = cli.Compact(ctx, rev, clientv3.WithCompactPhysical())
	cancel()
	return err
}

// IsLeader returns true if this member is the current cluster leader.
func (m *Member) IsLeader() (bool, error) {
	cli, err := m.CreateEtcdClient()
	if err != nil {
		return false, fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	defer cli.Close()

	resp, err := cli.Status(context.Background(), m.EtcdClientEndpoint)
	if err != nil {
		return false, err
	}
	return resp.Header.MemberId == resp.Leader, nil
}

// WriteHealthKey writes a health key to this member.
func (m *Member) WriteHealthKey() error {
	cli, err := m.CreateEtcdClient()
	if err != nil {
		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	defer cli.Close()

	// give enough time-out in case expensive requests (range/delete) are pending
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	_, err = cli.Put(ctx, "health", "good")
	cancel()
	if err != nil {
		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}
	return nil
}

// SaveSnapshot downloads a snapshot file from this member, locally.
// It's meant to requested remotely, so that local member can store
// snapshot file on local disk.
func (m *Member) SaveSnapshot(lg *zap.Logger) (err error) {
	// remove existing snapshot first
	if err = os.RemoveAll(m.SnapshotPath); err != nil {
		return err
	}

	var ccfg *clientv3.Config
	ccfg, err = m.CreateEtcdClientConfig()
	if err != nil {
		return fmt.Errorf("%v (%q)", err, m.EtcdClientEndpoint)
	}

	lg.Info(
		"snapshot save START",
		zap.String("member-name", m.Etcd.Name),
		zap.Strings("member-client-urls", m.Etcd.AdvertiseClientURLs),
		zap.String("snapshot-path", m.SnapshotPath),
	)
	now := time.Now()
	mgr := snapshot.NewV3(lg)
	if err = mgr.Save(context.Background(), *ccfg, m.SnapshotPath); err != nil {
		return err
	}
	took := time.Since(now)

	var fi os.FileInfo
	fi, err = os.Stat(m.SnapshotPath)
	if err != nil {
		return err
	}
	var st snapshot.Status
	st, err = mgr.Status(m.SnapshotPath)
	if err != nil {
		return err
	}
	m.SnapshotInfo = &SnapshotInfo{
		MemberName:        m.Etcd.Name,
		MemberClientURLs:  m.Etcd.AdvertiseClientURLs,
		SnapshotPath:      m.SnapshotPath,
		SnapshotFileSize:  humanize.Bytes(uint64(fi.Size())),
		SnapshotTotalSize: humanize.Bytes(uint64(st.TotalSize)),
		SnapshotTotalKey:  int64(st.TotalKey),
		SnapshotHash:      int64(st.Hash),
		SnapshotRevision:  st.Revision,
		Took:              fmt.Sprintf("%v", took),
	}
	lg.Info(
		"snapshot save END",
		zap.String("member-name", m.SnapshotInfo.MemberName),
		zap.Strings("member-client-urls", m.SnapshotInfo.MemberClientURLs),
		zap.String("snapshot-path", m.SnapshotPath),
		zap.String("snapshot-file-size", m.SnapshotInfo.SnapshotFileSize),
		zap.String("snapshot-total-size", m.SnapshotInfo.SnapshotTotalSize),
		zap.Int64("snapshot-total-key", m.SnapshotInfo.SnapshotTotalKey),
		zap.Int64("snapshot-hash", m.SnapshotInfo.SnapshotHash),
		zap.Int64("snapshot-revision", m.SnapshotInfo.SnapshotRevision),
		zap.String("took", m.SnapshotInfo.Took),
	)
	return nil
}

// RestoreSnapshot restores a cluster from a given snapshot file on disk.
// It's meant to requested remotely, so that local member can load the
// snapshot file from local disk.
func (m *Member) RestoreSnapshot(lg *zap.Logger) (err error) {
	if err = os.RemoveAll(m.EtcdOnSnapshotRestore.DataDir); err != nil {
		return err
	}
	if err = os.RemoveAll(m.EtcdOnSnapshotRestore.WALDir); err != nil {
		return err
	}

	lg.Info(
		"snapshot restore START",
		zap.String("member-name", m.Etcd.Name),
		zap.Strings("member-client-urls", m.Etcd.AdvertiseClientURLs),
		zap.String("snapshot-path", m.SnapshotPath),
	)
	now := time.Now()
	mgr := snapshot.NewV3(lg)
	err = mgr.Restore(snapshot.RestoreConfig{
		SnapshotPath:        m.SnapshotInfo.SnapshotPath,
		Name:                m.EtcdOnSnapshotRestore.Name,
		OutputDataDir:       m.EtcdOnSnapshotRestore.DataDir,
		OutputWALDir:        m.EtcdOnSnapshotRestore.WALDir,
		PeerURLs:            m.EtcdOnSnapshotRestore.AdvertisePeerURLs,
		InitialCluster:      m.EtcdOnSnapshotRestore.InitialCluster,
		InitialClusterToken: m.EtcdOnSnapshotRestore.InitialClusterToken,
		SkipHashCheck:       false,
		// TODO: set SkipHashCheck it true, to recover from existing db file
	})
	took := time.Since(now)
	lg.Info(
		"snapshot restore END",
		zap.String("member-name", m.SnapshotInfo.MemberName),
		zap.Strings("member-client-urls", m.SnapshotInfo.MemberClientURLs),
		zap.String("snapshot-path", m.SnapshotPath),
		zap.String("snapshot-file-size", m.SnapshotInfo.SnapshotFileSize),
		zap.String("snapshot-total-size", m.SnapshotInfo.SnapshotTotalSize),
		zap.Int64("snapshot-total-key", m.SnapshotInfo.SnapshotTotalKey),
		zap.Int64("snapshot-hash", m.SnapshotInfo.SnapshotHash),
		zap.Int64("snapshot-revision", m.SnapshotInfo.SnapshotRevision),
		zap.String("took", took.String()),
		zap.Error(err),
	)
	return err
}