Merge pull request #4 from etcd-io/master

Rebase master

.travis.yml

@@ -6,7 +6,7 @@ sudo: required
 services: docker
 
 go:
-  - 1.13
+  - 1.13.1
   - tip
 
 notifications:
@@ -30,13 +30,13 @@ env:
 matrix:
   fast_finish: true
   allow_failures:
-    - go: 1.13
+    - go: 1.13.1
       env: TARGET=linux-amd64-grpcproxy
-    - go: 1.13
+    - go: 1.13.1
       env: TARGET=linux-amd64-coverage
     - go: tip
       env: TARGET=linux-amd64-fmt-unit-go-tip
-    - go: 1.13
+    - go: 1.13.1
       env: TARGET=linux-386-unit
   exclude:
     - go: tip
@@ -57,7 +57,7 @@ matrix:
       env: TARGET=linux-amd64-grpcproxy
     - go: tip
       env: TARGET=linux-amd64-coverage
-    - go: 1.13
+    - go: 1.13.1
       env: TARGET=linux-amd64-fmt-unit-go-tip
     - go: tip
       env: TARGET=linux-386-unit
@@ -75,7 +75,7 @@ script:
       linux-amd64-fmt)
         docker run --rm \
           --volume=`pwd`:/go/src/go.etcd.io/etcd gcr.io/etcd-development/etcd-test:go${TRAVIS_GO_VERSION} \
-          /bin/bash -c "GOARCH=amd64 PASSES='fmt dep' ./test"
+          /bin/bash -c "GOARCH=amd64 PASSES='fmt bom dep' ./test"
         ;;
       linux-amd64-integration-1-cpu)
         docker run --rm \

.words

@@ -95,6 +95,7 @@ jitter
 WithBackoff
 BackoffLinearWithJitter
 jitter
+WithDialer
 WithMax
 ServerStreams
 BidiStreams

CHANGELOG-3.1.md

@@ -10,7 +10,7 @@ The minimum recommended etcd versions to run in **production** are 3.1.11+, 3.2.
 
 ## [v3.1.21](https://github.com/etcd-io/etcd/releases/tag/v3.1.21) (2019-TBD)
 
-### etcdctl
+### etcdctl v3
 
 - [Strip out insecure endpoints from DNS SRV records when using discovery](https://github.com/etcd-io/etcd/pull/10443) with etcdctl v2
 - Add [`etcdctl endpoint health --write-out` support](https://github.com/etcd-io/etcd/pull/9540).

CHANGELOG-3.2.md

@@ -8,10 +8,16 @@ The minimum recommended etcd versions to run in **production** are 3.1.11+, 3.2.
 
 <hr>
 
+## [v3.2.28](https://github.com/etcd-io/etcd/releases/tag/v3.2.28) (2019-TBD)
 
-## [v3.2.27](https://github.com/etcd-io/etcd/releases/tag/v3.2.27) (2019-TBD)
+### Improved
+- Add `etcd --experimental-peer-skip-client-san-verification` to [skip verification of peer client address](https://github.com/etcd-io/etcd/pull/11195).
+
+<hr>
 
-### etcdctl
+## [v3.2.27](https://github.com/etcd-io/etcd/releases/tag/v3.2.27) (2019-09-17)
+
+### etcdctl v3
 
 - [Strip out insecure endpoints from DNS SRV records when using discovery](https://github.com/etcd-io/etcd/pull/10443) with etcdctl v2
 - Add [`etcdctl endpoint health --write-out` support](https://github.com/etcd-io/etcd/pull/9540).
@@ -19,6 +25,13 @@ The minimum recommended etcd versions to run in **production** are 3.1.11+, 3.2.
   - The command output is changed. Previously, if endpoint is unreachable, the command output is
   "\<endpoint\> is unhealthy: failed to connect: \<error message\>". This change unified the error message, all error types
   now have the same output "\<endpoint\> is unhealthy: failed to commit proposal: \<error message\>".
+- Fix [`etcdctl snapshot status` to not modify snapshot file](https://github.com/etcd-io/etcd/pull/11157).
+  - For example, start etcd `v3.3.10`
+  - Write some data
+  - Use etcdctl `v3.3.10` to save snapshot
+  - Somehow, upgrading Kubernetes fails, thus rolling back to previous version etcd `v3.2.24`
+  - Run etcdctl `v3.2.24` `snapshot status` against the snapshot file saved from `v3.3.10` server
+  - Run etcdctl `v3.2.24` `snapshot restore` fails with `"expected sha256 [12..."`
 
 ### Metrics, Monitoring
 
@@ -30,6 +43,11 @@ Note that any `etcd_debugging_*` metrics are experimental and subject to change.
 - Add [`etcd_debugging_mvcc_current_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric.
 - Add [`etcd_debugging_mvcc_compact_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric.
 
+### Go
+
+- Compile with [*Go 1.8.7*](https://golang.org/doc/devel/release.html#go1.8).
+
+
 <hr>
 
 

CHANGELOG-3.3.md

@@ -15,13 +15,9 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.15...v3.3.16) an
 
 **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**
 
-### Dependency
-
-- Upgrade [`github.com/coreos/bbolt`](https://github.com/etcd-io/bbolt/releases) from [**`v1.3.1-coreos.6`**](https://github.com/etcd-io/bbolt/releases/tag/v1.3.1-coreos.6) to [**`v1.3.3`**](https://github.com/etcd-io/bbolt/releases/tag/v1.3.3).
-
-### Go
+### Improved
 
-- Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes.
+- Add `etcd --experimental-peer-skip-client-san-verification` to [skip verification of peer client address](https://github.com/etcd-io/etcd/pull/11196).
 
 ### Metrics, Monitoring
 
@@ -32,6 +28,18 @@ Note that any `etcd_debugging_*` metrics are experimental and subject to change.
 - Add [`etcd_debugging_mvcc_current_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric.
 - Add [`etcd_debugging_mvcc_compact_revision`](https://github.com/etcd-io/etcd/pull/11126) Prometheus metric.
 
+### Dependency
+
+- Upgrade [`github.com/coreos/bbolt`](https://github.com/etcd-io/bbolt/releases) from [**`v1.3.1-coreos.6`**](https://github.com/etcd-io/bbolt/releases/tag/v1.3.1-coreos.6) to [**`v1.3.3`**](https://github.com/etcd-io/bbolt/releases/tag/v1.3.3).
+
+### etcdctl v3
+
+- Fix [`etcdctl member add`](https://github.com/etcd-io/etcd/pull/11194) command to prevent potential timeout.
+
+### Go
+
+- Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes.
+
 
 <hr>
 
@@ -208,7 +216,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.3.11...v3.3.12) an
 
 **Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.3 upgrade guide](https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrade_3_3.md).**
 
-### etcdctl
+### etcdctl v3
 
 - [Strip out insecure endpoints from DNS SRV records when using discovery](https://github.com/etcd-io/etcd/pull/10443) with etcdctl v2
 
@@ -772,6 +780,13 @@ See [security doc](https://github.com/etcd-io/etcd/blob/master/Documentation/op-
 - Enable [`clientv3.WithRequireLeader(context.Context)` for `watch`](https://github.com/etcd-io/etcd/pull/8672) command.
 - Print [`"del"` instead of `"delete"`](https://github.com/etcd-io/etcd/pull/8297) in `txn` interactive mode.
 - Print [`ETCD_INITIAL_ADVERTISE_PEER_URLS` in `member add`](https://github.com/etcd-io/etcd/pull/8332).
+- Fix [`etcdctl snapshot status` to not modify snapshot file](https://github.com/etcd-io/etcd/pull/8815).
+  - For example, start etcd `v3.3.10`
+  - Write some data
+  - Use etcdctl `v3.3.10` to save snapshot
+  - Somehow, upgrading Kubernetes fails, thus rolling back to previous version etcd `v3.2.24`
+  - Run etcdctl `v3.2.24` `snapshot status` against the snapshot file saved from `v3.3.10` server
+  - Run etcdctl `v3.2.24` `snapshot restore` fails with `"expected sha256 [12..."`
 
 ### etcdctl v3
 

CHANGELOG-3.4.md

@@ -9,7 +9,33 @@ The minimum recommended etcd versions to run in **production** are 3.1.11+, 3.2.
 <hr>
 
 
-## [v3.4.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.1) (2019-TBD)
+## [v3.4.2](https://github.com/etcd-io/etcd/releases/tag/v3.4.2) (2019 TBD)
+
+See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.1...v3.4.2) and [v3.4 upgrade guide](https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrade_3_4.md) for any breaking changes.
+
+**Again, before running upgrades from any previous release, please make sure to read change logs below and [v3.4 upgrade guide](https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrade_3_4.md).**
+
+### Dependency
+
+- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.23.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.1) to [**`v1.24.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.24.0).
+
+### etcdctl v3
+
+- Fix [`etcdctl member add`](https://github.com/etcd-io/etcd/pull/11194) command to prevent potential timeout.
+
+### etcdserver
+
+- Add [`tracing`](https://github.com/etcd-io/etcd/pull/11179) to range, put and compact requests in etcdserver.
+
+### Go
+
+- Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes.
+
+
+<hr>
+
+
+## [v3.4.1](https://github.com/etcd-io/etcd/releases/tag/v3.4.1) (2019-09-17)
 
 See [code changes](https://github.com/etcd-io/etcd/compare/v3.4.0...v3.4.1) and [v3.4 upgrade guide](https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrade_3_4.md) for any breaking changes.
 
@@ -29,6 +55,14 @@ Note that any `etcd_debugging_*` metrics are experimental and subject to change.
 - Fix [secure server logging message](https://github.com/etcd-io/etcd/commit/8b053b0f44c14ac0d9f39b9b78c17c57d47966eb).
 - Remove [redundant `%` characters in file descriptor warning message](https://github.com/etcd-io/etcd/commit/d5f79adc9cea9ec8c93669526464b0aa19ed417b).
 
+### Package `embed`
+
+- Add [`embed.Config.ZapLoggerBuilder`](https://github.com/etcd-io/etcd/pull/11148) to allow creating a custom zap logger.
+
+### Dependency
+
+- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.23.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.0) to [**`v1.23.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.1).
+
 ### Go
 
 - Compile with [*Go 1.12.9*](https://golang.org/doc/devel/release.html#go1.12) including [*Go 1.12.8*](https://groups.google.com/d/msg/golang-announce/65QixT3tcmg/DrFiG6vvCwAJ) security fixes.

CHANGELOG-3.5.md

@@ -80,6 +80,15 @@ Note that any `etcd_debugging_*` metrics are experimental and subject to change.
 
 - Remove [`embed.Config.Debug`](https://github.com/etcd-io/etcd/pull/10947).
   - Use `embed.Config.LogLevel` instead.
+- Add [`embed.Config.ZapLoggerBuilder`](https://github.com/etcd-io/etcd/pull/11147) to allow creating a custom zap logger.
+
+### Package `clientv3`
+
+- Add [TryLock](https://github.com/etcd-io/etcd/pull/11104) method to `clientv3/concurrency/Mutex`. A non-blocking method on `Mutex` which does not wait to get lock on the Mutex, returns immediately if Mutex is locked by another session.
+
+### etcdctl v3
+
+- Fix [`etcdctl member add`](https://github.com/etcd-io/etcd/pull/11194) command to prevent potential timeout.
 
 ### gRPC gateway
 
@@ -87,11 +96,18 @@ Note that any `etcd_debugging_*` metrics are experimental and subject to change.
   - Deprecated [`/v3beta`](https://github.com/etcd-io/etcd/pull/9298).
   - `curl -L http://localhost:2379/v3beta/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` does work in v3.5. Use `curl -L http://localhost:2379/v3/kv/put -X POST -d '{"key": "Zm9v", "value": "YmFy"}'` instead.
 
+### Dependency
+
+- Upgrade [`google.golang.org/grpc`](https://github.com/grpc/grpc-go/releases) from [**`v1.23.0`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.0) to [**`v1.23.1`**](https://github.com/grpc/grpc-go/releases/tag/v1.23.1).
+
 ### Go
 
 - Require [*Go 1.13+*](https://github.com/etcd-io/etcd/pull/11110).
 - Compile with [*Go 1.13*](https://golang.org/doc/devel/release.html#go1.13)
 
+### Project Governance
+
+- The etcd team has added, a well defined and openly discussed, project [governance](https://github.com/etcd-io/etcd/pull/11175).
 
 <hr>
 

CONTRIBUTING.md

@@ -6,6 +6,7 @@ etcd is Apache 2.0 licensed and accepts contributions via GitHub pull requests.
 
 - Email: [etcd-dev](https://groups.google.com/forum/?hl=en#!forum/etcd-dev)
 - IRC: #[etcd](irc://irc.freenode.org:6667/#etcd) IRC channel on freenode.org
+- Slack: [#etcd](https://kubernetes.slack.com/messages/C3HD8ARJ5/details/)
 
 ## Getting started
 

Documentation/op-guide/configuration.md

@@ -411,7 +411,7 @@ Follow the instructions when using these flags.
 + env variable: ETCD_ENABLE_PPROF
 
 ### --metrics
-+ Set level of detail for exported metrics, specify 'extensive' to include histogram metrics.
++ Set level of detail for exported metrics, specify 'extensive' to include server side grpc histogram metrics.
 + default: basic
 + env variable: ETCD_METRICS
 

GOVERNANCE.md

@@ -0,0 +1,80 @@
+# etcd Governance
+
+## Principles
+
+The etcd community adheres to the following principles:
+
+- Open: etcd is open source.
+- Welcoming and respectful: See [Code of Conduct](code-of-conduct.md).
+- Transparent and accessible: Changes to the etcd code repository and CNCF related
+activities (e.g. level, involvement, etc) are done in public.
+- Merit: Ideas and contributions are accepted according to their technical merit for
+the betterment of the project. For specific guidance on practical contribution steps
+please see [CONTRIBUTING](./CONTRIBUTING.md) guide.
+
+## Maintainers
+
+[Maintainers](./MAINTAINERS) are first and foremost contributors that have shown they
+are committed to the long term success of a project. Maintainership is about building
+trust with the current maintainers of the project and being a person that they can
+depend on to make decisions in the best interest of the project in a consistent manner.
+The maintainers role can be a top-level or restricted to certain package/feature
+depending upon their commitment in fulfilling the expected responsibilities as explained
+below.
+
+### Top-level maintainer
+
+- Running the etcd release processes
+- Ownership of test and debug infrastructure
+- Triage GitHub issues to keep the issue count low (goal: under 100)
+- Regularly review GitHub pull requests across all pkgs
+- Providing cross pkg design review
+- Monitor email aliases
+- Participate when called upon in the [security disclosure and release process](security/README.md)
+- General project maintenance
+
+### Package/feature maintainer
+
+- Ownership of test and debug failures in a pkg/feature
+- Resolution of bugs triaged to a package/feature
+- Regularly review pull requests to the pkg subsystem
+
+Contributors who are interested in becoming a maintainer, if performing these
+responsibilities, should discuss their interest with the existing maintainers. New
+maintainers must be nominated by an existing maintainer and must be elected by a
+supermajority of maintainers. Likewise, maintainers can be removed by a supermajority
+of the maintainers and moved to emeritus status.
+
+Life priorities, interests, and passions can change. If a maintainer needs to step
+down, inform other maintainers about this intention, and if possible, help find someone
+to pick up the related work. At the very least, ensure the related work can be continued.
+Afterward, create a pull request to remove yourself from the [MAINTAINERS](./MAINTAINERS)
+file.
+
+## Reviewers
+
+[Reviewers](./MAINTAINERS) are contributors who have demonstrated greater skill in
+reviewing the code contribution from other contributors. Their LGTM counts towards
+merging a code change into the project. A reviewer is generally on the ladder towards
+maintainership. New reviewers must be nominated by an existing maintainer and must be
+elected by a supermajority of maintainers. Likewise, reviewers can be removed by a
+supermajority of the  maintainers or can resign by notifying the maintainers.
+
+## Decision making process
+
+Decisions are built on consensus between maintainers publicly. Proposals and ideas
+can either be submitted for agreement via a GitHub issue or PR, or by sending an email
+to `etcd-maintainers@googlegroups.com`.
+
+## Conflict resolution
+
+In general, we prefer that technical issues and maintainer membership are amicably
+worked out between the persons involved. However, any technical dispute that has
+reached an impasse with a subset of the community, any contributor may open a GitHub
+issue or PR or send an email to `etcd-maintainers@googlegroups.com`. If the
+maintainers themselves cannot decide an issue, the issue will be resolved by a
+supermajority of the maintainers.
+
+## Changes in Governance
+
+Changes in project governance could be initiated by opening a GitHub PR.

MAINTAINERS

@@ -1,4 +1,6 @@
-# This is the official list of etcd maintainers.
+# The official list of maintainers and reviewers for the project maintenance.
+#
+# Refer to the GOVERNANCE.md for description of the roles.
 #
 # Names should be added to this file like so:
 #     Individual's name <submission email address> (@GITHUB_HANDLE) pkg:*
@@ -6,6 +8,7 @@
 #
 # Please keep the list sorted.
 
+# MAINTAINERS
 Brandon Philips <bphilips@redhat.com> (@philips) pkg:*
 Gyuho Lee <gyuhox@gmail.com> <leegyuho@amazon.com> (@gyuho) pkg:*
 Hitoshi Mitake <h.mitake@gmail.com> (@mitake) pkg:*
@@ -18,3 +21,5 @@ Xiang Li <xiangli.cs@gmail.com> (@xiang90) pkg:*
 Ben Darnell <ben@cockroachlabs.com> (@bdarnell) pkg:go.etcd.io/etcd/raft
 Tobias Grieger <tobias.schottdorf@gmail.com> (@tbg) pkg:go.etcd.io/etcd/raft
 
+# REVIEWERS
+Wenjia Zhang <wenjiazhang@google.com> (@wenjiaswe) pkg:*

MAINTAINERS_RULES.md

@@ -1,16 +0,0 @@
-
-This document describes basic expectations for maintainers. To become a maintainer, start taking on these responsibilities. Consistent contributors then discuss with existing maintainers to become the official [MAINTAINERS](./MAINTAINERS).
-
-### Top-level maintainer
-
-- Running the etcd release processes
-- Ownership of test and debug infrastructure
-- Resolve or redirect issues to keep the issue count low (goal: under 100)
-- Regularly review pull requests across all pkgs
-- Providing cross pkg design review
-
-### Package/feature maintainer
-
-- Ownership of test and debug failures in a pkg/feature
-- Resolution of bugs triaged to a package/feature
-- Regularly review pull requests to the pkg subsystem

Makefile

@@ -51,7 +51,7 @@ docker-remove:
 
 
 
-GO_VERSION ?= 1.13
+GO_VERSION ?= 1.13.1
 ETCD_VERSION ?= $(shell git rev-parse --short HEAD || echo "GitNotFound")
 
 TEST_SUFFIX = $(shell date +%s | base64 | head -c 15)
@@ -65,11 +65,11 @@ endif
 
 
 # Example:
-#   GO_VERSION=1.13 make build-docker-test
+#   GO_VERSION=1.13.1 make build-docker-test
 #   make build-docker-test
 #
 #   gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd-development.json)" https://gcr.io
-#   GO_VERSION=1.13 make push-docker-test
+#   GO_VERSION=1.13.1 make push-docker-test
 #   make push-docker-test
 #
 #   gsutil -m acl ch -u allUsers:R -r gs://artifacts.etcd-development.appspot.com
@@ -281,6 +281,7 @@ docker-static-ip-test-certs-metrics-proxy-run:
 #   make docker-dns-test-certs-wildcard-run
 #   make docker-dns-test-certs-common-name-auth-run
 #   make docker-dns-test-certs-common-name-multi-run
+#   make docker-dns-test-certs-san-dns-run
 
 build-docker-dns-test:
 	$(info GO_VERSION: $(GO_VERSION))
@@ -389,6 +390,19 @@ docker-dns-test-certs-common-name-multi-run:
 	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
 	  /bin/bash -c "cd /etcd && /certs-common-name-multi/run.sh && rm -rf m*.etcd"
 
+docker-dns-test-certs-san-dns-run:
+	$(info GO_VERSION: $(GO_VERSION))
+	$(info HOST_TMP_DIR: $(HOST_TMP_DIR))
+	$(info TMP_DIR_MOUNT_FLAG: $(TMP_DIR_MOUNT_FLAG))
+	docker run \
+	  --rm \
+	  --tty \
+	  --dns 127.0.0.1 \
+	  $(TMP_DIR_MOUNT_FLAG) \
+	  --mount type=bind,source=`pwd`/bin,destination=/etcd \
+	  --mount type=bind,source=`pwd`/tests/docker-dns/certs-san-dns,destination=/certs-san-dns \
+	  gcr.io/etcd-development/etcd-dns-test:go$(GO_VERSION) \
+	  /bin/bash -c "cd /etcd && /certs-san-dns/run.sh && rm -rf m*.etcd"
 
 
 # Example:

NOTICE

@@ -1,5 +0,0 @@
-CoreOS Project
-Copyright 2014 CoreOS, Inc
-
-This product includes software developed at CoreOS, Inc.
-(http://www.coreos.com/).

OWNERS

@@ -1,20 +0,0 @@
-approvers:
-- heyitsanthony
-- philips
-- fanminshi
-- gyuho
-- mitake
-- jpbetz
-- xiang90
-- hexfusion
-reviewers:
-- heyitsanthony
-- philips
-- fanminshi
-- gyuho
-- mitake
-- jpbetz
-- xiang90
-- wenjiaswe
-- jingyih
-- hexfusion

README.md

@@ -52,16 +52,13 @@ Time:
 - [Jul 11th, 2019 11:00 AM video](https://youtu.be/k_FZEipWD6Y)
 - [Jul 25, 2019 11:00 AM video](https://youtu.be/VSUJTACO93I)
 - [Aug 22, 2019 11:00 AM video](https://youtu.be/6IBQ-VxQmuM) 
-- Sep 19, 2019 11:00 AM
+- [Sep 19, 2019 11:00 AM video](https://youtu.be/SqfxU9DhBOc)
 - Nov 14, 2019 11:00 AM
 - Dec 12, 2019 11:00 AM
 
-Join Hangouts Meet
-meet.google.com/umg-nrxn-qvs
+Join Hangouts Meet: [meet.google.com/umg-nrxn-qvs](https://meet.google.com/umg-nrxn-qvs)
 
-Join by phone
-‪+1 405-792-0633‬ PIN: ‪299 906‬#
-More phone numbers
+Join by phone: +1 405-792-0633‬ PIN: ‪299 906‬#
 
 
 ## Getting started
@@ -72,7 +69,7 @@ The easiest way to get etcd is to use one of the pre-built release binaries whic
 
 For more installation guides, please check out [play.etcd.io](http://play.etcd.io) and [operating etcd](https://github.com/etcd-io/etcd/tree/master/Documentation#operating-etcd-clusters).
 
-For those wanting to try the very latest version, [build the latest version of etcd][dl-build] from the `master` branch. This first needs [*Go*](https://golang.org/) installed (version 1.12+ is required). All development occurs on `master`, including new features and bug fixes. Bug fixes are first targeted at `master` and subsequently ported to release branches, as described in the [branch management][branch-management] guide.
+For those wanting to try the very latest version, [build the latest version of etcd][dl-build] from the `master` branch. This first needs [*Go*](https://golang.org/) installed (version 1.13+ is required). All development occurs on `master`, including new features and bug fixes. Bug fixes are first targeted at `master` and subsequently ported to release branches, as described in the [branch management][branch-management] guide.
 
 [github-release]: https://github.com/etcd-io/etcd/releases
 [branch-management]: ./Documentation/branch_management.md

clientv3/balancer/resolver/endpoint/endpoint.go

@@ -16,7 +16,9 @@
 package endpoint
 
 import (
+	"context"
 	"fmt"
+	"net"
 	"net/url"
 	"strings"
 	"sync"
@@ -228,13 +230,18 @@ func ParseTarget(target string) (string, string, error) {
 	return parts[0], parts[1], nil
 }
 
-// ParseHostPort splits a "<host>:<port>" string into the host and port parts.
-// The port part is optional.
-func ParseHostPort(hostPort string) (host string, port string) {
-	parts := strings.SplitN(hostPort, ":", 2)
-	host = parts[0]
-	if len(parts) > 1 {
-		port = parts[1]
+// Dialer dials a endpoint using net.Dialer.
+// Context cancelation and timeout are supported.
+func Dialer(ctx context.Context, dialEp string) (net.Conn, error) {
+	proto, host, _ := ParseEndpoint(dialEp)
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
+	}
+	dialer := &net.Dialer{}
+	if deadline, ok := ctx.Deadline(); ok {
+		dialer.Deadline = deadline
 	}
-	return host, port
+	return dialer.DialContext(ctx, proto, host)
 }

clientv3/client.go

@@ -230,24 +230,17 @@ func (c *Client) dialSetupOpts(creds grpccredentials.TransportCredentials, dopts
 	}
 	opts = append(opts, dopts...)
 
-	// Provide a net dialer that supports cancelation and timeout.
-	f := func(dialEp string, t time.Duration) (net.Conn, error) {
-		proto, host, _ := endpoint.ParseEndpoint(dialEp)
-		select {
-		case <-c.ctx.Done():
-			return nil, c.ctx.Err()
-		default:
-		}
-		dialer := &net.Dialer{Timeout: t}
-		return dialer.DialContext(c.ctx, proto, host)
-	}
-	opts = append(opts, grpc.WithDialer(f))
-
+	dialer := endpoint.Dialer
 	if creds != nil {
 		opts = append(opts, grpc.WithTransportCredentials(creds))
+		// gRPC load balancer workaround. See credentials.transportCredential for details.
+		if credsDialer, ok := creds.(TransportCredentialsWithDialer); ok {
+			dialer = credsDialer.Dialer
+		}
 	} else {
 		opts = append(opts, grpc.WithInsecure())
 	}
+	opts = append(opts, grpc.WithContextDialer(dialer))
 
 	// Interceptor retry and backoff.
 	// TODO: Replace all of clientv3/retry.go with interceptor based retry, or with
@@ -266,7 +259,10 @@ func (c *Client) dialSetupOpts(creds grpccredentials.TransportCredentials, dopts
 
 // Dial connects to a single endpoint using the client's config.
 func (c *Client) Dial(ep string) (*grpc.ClientConn, error) {
-	creds := c.directDialCreds(ep)
+	creds, err := c.directDialCreds(ep)
+	if err != nil {
+		return nil, err
+	}
 	// Use the grpc passthrough resolver to directly dial a single endpoint.
 	// This resolver passes through the 'unix' and 'unixs' endpoints schemes used
 	// by etcd without modification, allowing us to directly dial endpoints and
@@ -369,8 +365,8 @@ func (c *Client) dial(target string, creds grpccredentials.TransportCredentials,
 	return conn, nil
 }
 
-func (c *Client) directDialCreds(ep string) grpccredentials.TransportCredentials {
-	_, hostPort, scheme := endpoint.ParseEndpoint(ep)
+func (c *Client) directDialCreds(ep string) (grpccredentials.TransportCredentials, error) {
+	_, host, scheme := endpoint.ParseEndpoint(ep)
 	creds := c.creds
 	if len(scheme) != 0 {
 		creds = c.processCreds(scheme)
@@ -379,12 +375,17 @@ func (c *Client) directDialCreds(ep string) grpccredentials.TransportCredentials
 			// Set the server name must to the endpoint hostname without port since grpc
 			// otherwise attempts to check if x509 cert is valid for the full endpoint
 			// including the scheme and port, which fails.
-			host, _ := endpoint.ParseHostPort(hostPort)
-			clone.OverrideServerName(host)
+			overrideServerName, _, err := net.SplitHostPort(host)
+			if err != nil {
+				// Either the host didn't have a port or the host could not be parsed. Either way, continue with the
+				// original host string.
+				overrideServerName = host
+			}
+			clone.OverrideServerName(overrideServerName)
 			creds = clone
 		}
 	}
-	return creds
+	return creds, nil
 }
 
 func (c *Client) dialWithBalancerCreds(ep string) grpccredentials.TransportCredentials {
@@ -663,3 +664,9 @@ func IsConnCanceled(err error) bool {
 	// <= gRPC v1.7.x returns 'errors.New("grpc: the client connection is closing")'
 	return strings.Contains(err.Error(), "grpc: the client connection is closing")
 }
+
+// TransportCredentialsWithDialer is for a gRPC load balancer workaround. See credentials.transportCredential for details.
+type TransportCredentialsWithDialer interface {
+	grpccredentials.TransportCredentials
+	Dialer(ctx context.Context, dialEp string) (net.Conn, error)
+}

clientv3/concurrency/example_mutex_test.go

@@ -23,6 +23,57 @@ import (
 	"go.etcd.io/etcd/clientv3/concurrency"
 )
 
+func ExampleMutex_TryLock() {
+	cli, err := clientv3.New(clientv3.Config{Endpoints: endpoints})
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer cli.Close()
+
+	// create two separate sessions for lock competition
+	s1, err := concurrency.NewSession(cli)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer s1.Close()
+	m1 := concurrency.NewMutex(s1, "/my-lock")
+
+	s2, err := concurrency.NewSession(cli)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer s2.Close()
+	m2 := concurrency.NewMutex(s2, "/my-lock")
+
+	// acquire lock for s1
+	if err = m1.Lock(context.TODO()); err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println("acquired lock for s1")
+
+	if err = m2.TryLock(context.TODO()); err == nil {
+		log.Fatal("should not acquire lock")
+	}
+	if err == concurrency.ErrLocked {
+		fmt.Println("cannot acquire lock for s2, as already locked in another session")
+	}
+
+	if err = m1.Unlock(context.TODO()); err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println("released lock for s1")
+	if err = m2.TryLock(context.TODO()); err != nil {
+		log.Fatal(err)
+	}
+	fmt.Println("acquired lock for s2")
+
+	// Output:
+	// acquired lock for s1
+	// cannot acquire lock for s2, as already locked in another session
+	// released lock for s1
+	// acquired lock for s2
+}
+
 func ExampleMutex_Lock() {
 	cli, err := clientv3.New(clientv3.Config{Endpoints: endpoints})
 	if err != nil {

clientv3/concurrency/mutex.go

@@ -16,6 +16,7 @@ package concurrency
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"sync"
 
@@ -23,6 +24,9 @@ import (
 	pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
 )
 
+// ErrLocked is returned by TryLock when Mutex is already locked by another session.
+var ErrLocked = errors.New("mutex: Locked by another session")
+
 // Mutex implements the sync Locker interface with etcd
 type Mutex struct {
 	s *Session
@@ -37,35 +41,44 @@ func NewMutex(s *Session, pfx string) *Mutex {
 	return &Mutex{s, pfx + "/", "", -1, nil}
 }
 
+// TryLock locks the mutex if not already locked by another session.
+// If lock is held by another session, return immediately after attempting necessary cleanup
+// The ctx argument is used for the sending/receiving Txn RPC.
+func (m *Mutex) TryLock(ctx context.Context) error {
+	resp, err := m.tryAcquire(ctx)
+	if err != nil {
+		return err
+	}
+	// if no key on prefix / the minimum rev is key, already hold the lock
+	ownerKey := resp.Responses[1].GetResponseRange().Kvs
+	if len(ownerKey) == 0 || ownerKey[0].CreateRevision == m.myRev {
+		m.hdr = resp.Header
+		return nil
+	}
+	client := m.s.Client()
+	// Cannot lock, so delete the key
+	if _, err := client.Delete(ctx, m.myKey); err != nil {
+		return err
+	}
+	m.myKey = "\x00"
+	m.myRev = -1
+	return ErrLocked
+}
+
 // Lock locks the mutex with a cancelable context. If the context is canceled
 // while trying to acquire the lock, the mutex tries to clean its stale lock entry.
 func (m *Mutex) Lock(ctx context.Context) error {
-	s := m.s
-	client := m.s.Client()
-
-	m.myKey = fmt.Sprintf("%s%x", m.pfx, s.Lease())
-	cmp := v3.Compare(v3.CreateRevision(m.myKey), "=", 0)
-	// put self in lock waiters via myKey; oldest waiter holds lock
-	put := v3.OpPut(m.myKey, "", v3.WithLease(s.Lease()))
-	// reuse key in case this session already holds the lock
-	get := v3.OpGet(m.myKey)
-	// fetch current holder to complete uncontended path with only one RPC
-	getOwner := v3.OpGet(m.pfx, v3.WithFirstCreate()...)
-	resp, err := client.Txn(ctx).If(cmp).Then(put, getOwner).Else(get, getOwner).Commit()
+	resp, err := m.tryAcquire(ctx)
 	if err != nil {
 		return err
 	}
-	m.myRev = resp.Header.Revision
-	if !resp.Succeeded {
-		m.myRev = resp.Responses[0].GetResponseRange().Kvs[0].CreateRevision
-	}
 	// if no key on prefix / the minimum rev is key, already hold the lock
 	ownerKey := resp.Responses[1].GetResponseRange().Kvs
 	if len(ownerKey) == 0 || ownerKey[0].CreateRevision == m.myRev {
 		m.hdr = resp.Header
 		return nil
 	}
-
+	client := m.s.Client()
 	// wait for deletion revisions prior to myKey
 	hdr, werr := waitDeletes(ctx, client, m.pfx, m.myRev-1)
 	// release lock key if wait failed
@@ -77,6 +90,29 @@ func (m *Mutex) Lock(ctx context.Context) error {
 	return werr
 }
 
+func (m *Mutex) tryAcquire(ctx context.Context) (*v3.TxnResponse, error) {
+	s := m.s
+	client := m.s.Client()
+
+	m.myKey = fmt.Sprintf("%s%x", m.pfx, s.Lease())
+	cmp := v3.Compare(v3.CreateRevision(m.myKey), "=", 0)
+	// put self in lock waiters via myKey; oldest waiter holds lock
+	put := v3.OpPut(m.myKey, "", v3.WithLease(s.Lease()))
+	// reuse key in case this session already holds the lock
+	get := v3.OpGet(m.myKey)
+	// fetch current holder to complete uncontended path with only one RPC
+	getOwner := v3.OpGet(m.pfx, v3.WithFirstCreate()...)
+	resp, err := client.Txn(ctx).If(cmp).Then(put, getOwner).Else(get, getOwner).Commit()
+	if err != nil {
+		return nil, err
+	}
+	m.myRev = resp.Header.Revision
+	if !resp.Succeeded {
+		m.myRev = resp.Responses[0].GetResponseRange().Kvs[0].CreateRevision
+	}
+	return resp, nil
+}
+
 func (m *Mutex) Unlock(ctx context.Context) error {
 	client := m.s.Client()
 	if _, err := client.Delete(ctx, m.myKey); err != nil {

clientv3/credentials/credentials.go

@@ -22,6 +22,7 @@ import (
 	"net"
 	"sync"
 
+	"go.etcd.io/etcd/clientv3/balancer/resolver/endpoint"
 	"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
 	grpccredentials "google.golang.org/grpc/credentials"
 )
@@ -65,38 +66,37 @@ func (b *bundle) NewWithMode(mode string) (grpccredentials.Bundle, error) {
 }
 
 // transportCredential implements "grpccredentials.TransportCredentials" interface.
+// transportCredential wraps TransportCredentials to track which
+// addresses are dialed for which endpoints, and then sets the authority when checking the endpoint's cert to the
+// hostname or IP of the dialed endpoint.
+// This is a workaround of a gRPC load balancer issue. gRPC uses the dialed target's service name as the authority when
+// checking all endpoint certs, which does not work for etcd servers using their hostname or IP as the Subject Alternative Name
+// in their TLS certs.
+// To enable, include both WithTransportCredentials(creds) and WithContextDialer(creds.Dialer)
+// when dialing.
 type transportCredential struct {
 	gtc grpccredentials.TransportCredentials
+	mu  sync.Mutex
+	// addrToEndpoint maps from the connection addresses that are dialed to the hostname or IP of the
+	// endpoint provided to the dialer when dialing
+	addrToEndpoint map[string]string
 }
 
 func newTransportCredential(cfg *tls.Config) *transportCredential {
 	return &transportCredential{
-		gtc: grpccredentials.NewTLS(cfg),
+		gtc:            grpccredentials.NewTLS(cfg),
+		addrToEndpoint: map[string]string{},
 	}
 }
 
 func (tc *transportCredential) ClientHandshake(ctx context.Context, authority string, rawConn net.Conn) (net.Conn, grpccredentials.AuthInfo, error) {
-	// Only overwrite when authority is an IP address!
-	// Let's say, a server runs SRV records on "etcd.local" that resolves
-	// to "m1.etcd.local", and its SAN field also includes "m1.etcd.local".
-	// But what if SAN does not include its resolved IP address (e.g. 127.0.0.1)?
-	// Then, the server should only authenticate using its DNS hostname "m1.etcd.local",
-	// instead of overwriting it with its IP address.
-	// And we do not overwrite "localhost" either. Only overwrite IP addresses!
-	if isIP(authority) {
-		target := rawConn.RemoteAddr().String()
-		if authority != target {
-			// When user dials with "grpc.WithDialer", "grpc.DialContext" "cc.parsedTarget"
-			// update only happens once. This is problematic, because when TLS is enabled,
-			// retries happen through "grpc.WithDialer" with static "cc.parsedTarget" from
-			// the initial dial call.
-			// If the server authenticates by IP addresses, we want to set a new endpoint as
-			// a new authority. Otherwise
-			// "transport: authentication handshake failed: x509: certificate is valid for 127.0.0.1, 192.168.121.180, not 192.168.223.156"
-			// when the new dial target is "192.168.121.180" whose certificate host name is also "192.168.121.180"
-			// but client tries to authenticate with previously set "cc.parsedTarget" field "192.168.223.156"
-			authority = target
-		}
+	// Set the authority when checking the endpoint's cert to the hostname or IP of the dialed endpoint
+	tc.mu.Lock()
+	dialEp, ok := tc.addrToEndpoint[rawConn.RemoteAddr().String()]
+	tc.mu.Unlock()
+	if ok {
+		_, host, _ := endpoint.ParseEndpoint(dialEp)
+		authority = host
 	}
 	return tc.gtc.ClientHandshake(ctx, authority, rawConn)
 }
@@ -115,8 +115,15 @@ func (tc *transportCredential) Info() grpccredentials.ProtocolInfo {
 }
 
 func (tc *transportCredential) Clone() grpccredentials.TransportCredentials {
+	copy := map[string]string{}
+	tc.mu.Lock()
+	for k, v := range tc.addrToEndpoint {
+		copy[k] = v
+	}
+	tc.mu.Unlock()
 	return &transportCredential{
-		gtc: tc.gtc.Clone(),
+		gtc:            tc.gtc.Clone(),
+		addrToEndpoint: copy,
 	}
 }
 
@@ -124,6 +131,17 @@ func (tc *transportCredential) OverrideServerName(serverNameOverride string) err
 	return tc.gtc.OverrideServerName(serverNameOverride)
 }
 
+func (tc *transportCredential) Dialer(ctx context.Context, dialEp string) (net.Conn, error) {
+	// Keep track of which addresses are dialed for which endpoints
+	conn, err := endpoint.Dialer(ctx, dialEp)
+	if conn != nil {
+		tc.mu.Lock()
+		tc.addrToEndpoint[conn.RemoteAddr().String()] = dialEp
+		tc.mu.Unlock()
+	}
+	return conn, err
+}
+
 // perRPCCredential implements "grpccredentials.PerRPCCredentials" interface.
 type perRPCCredential struct {
 	authToken   string

clientv3/example_kv_test.go

@@ -249,7 +249,6 @@ func ExampleKV_txn() {
 	}
 
 	gresp, err := kvc.Get(context.TODO(), "key")
-	cancel()
 	if err != nil {
 		log.Fatal(err)
 	}

clientv3/integration/cluster_test.go

@@ -276,8 +276,7 @@ func TestMemberPromote(t *testing.T) {
 		select {
 		case <-time.After(500 * time.Millisecond):
 		case <-timeout:
-			t.Errorf("failed all attempts to promote learner member, last error: %v", err)
-			break
+			t.Fatalf("failed all attempts to promote learner member, last error: %v", err)
 		}
 
 		_, err = capi.MemberPromote(context.Background(), learnerID)

clientv3/snapshot/v3_snapshot.go

@@ -39,6 +39,7 @@ import (
 	"go.etcd.io/etcd/mvcc"
 	"go.etcd.io/etcd/mvcc/backend"
 	"go.etcd.io/etcd/pkg/fileutil"
+	"go.etcd.io/etcd/pkg/traceutil"
 	"go.etcd.io/etcd/pkg/types"
 	"go.etcd.io/etcd/raft"
 	"go.etcd.io/etcd/raft/raftpb"
@@ -384,7 +385,7 @@ func (s *v3Manager) saveDB() error {
 	lessor := lease.NewLessor(s.lg, be, lease.LessorConfig{MinLeaseTTL: math.MaxInt64})
 
 	mvs := mvcc.NewStore(s.lg, be, lessor, (*initIndex)(&commit), mvcc.StoreConfig{CompactionBatchLimit: math.MaxInt32})
-	txn := mvs.Write()
+	txn := mvs.Write(traceutil.TODO())
 	btx := be.BatchTx()
 	del := func(k, v []byte) error {
 		txn.DeleteRange(k, nil)

code-of-conduct.md

@@ -1,61 +1,3 @@
-## CoreOS Community Code of Conduct
+## etcd Community Code of Conduct
 
-### Contributor Code of Conduct
-
-As contributors and maintainers of this project, and in the interest of
-fostering an open and welcoming community, we pledge to respect all people who
-contribute through reporting issues, posting feature requests, updating
-documentation, submitting pull requests or patches, and other activities.
-
-We are committed to making participation in this project a harassment-free
-experience for everyone, regardless of level of experience, gender, gender
-identity and expression, sexual orientation, disability, personal appearance,
-body size, race, ethnicity, age, religion, or nationality.
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery
-* Personal attacks
-* Trolling or insulting/derogatory comments
-* Public or private harassment
-* Publishing others' private information, such as physical or electronic addresses, without explicit permission
-* Other unethical or unprofessional conduct.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct. By adopting this Code of Conduct,
-project maintainers commit themselves to fairly and consistently applying these
-principles to every aspect of managing this project. Project maintainers who do
-not follow or enforce the Code of Conduct may be permanently removed from the
-project team.
-
-This code of conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community.
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting a project maintainer, Brandon Philips
-<brandon.philips@coreos.com>, and/or Rithu John <rithu.john@coreos.com>.
-
-This Code of Conduct is adapted from the Contributor Covenant
-(http://contributor-covenant.org), version 1.2.0, available at
-http://contributor-covenant.org/version/1/2/0/
-
-### CoreOS Events Code of Conduct
-
-CoreOS events are working conferences intended for professional networking and
-collaboration in the CoreOS community. Attendees are expected to behave
-according to professional standards and in accordance with their employer’s
-policies on appropriate workplace behavior.
-
-While at CoreOS events or related social networking opportunities, attendees
-should not engage in discriminatory or offensive speech or actions including
-but not limited to gender, sexuality, race, age, disability, or religion.
-Speakers should be especially aware of these concerns.
-
-CoreOS does not condone any statements by speakers contrary to these standards.
-CoreOS reserves the right to deny entrance and/or eject from an event (without
-refund) any individual found to be engaging in discriminatory or offensive
-speech or actions.
-
-Please bring any concerns to the immediate attention of designated on-site
-staff, Brandon Philips <brandon.philips@coreos.com>, and/or Rithu John <rithu.john@coreos.com>.
+etcd follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).

embed/config.go

@@ -303,8 +303,8 @@ type Config struct {
 	// It can be multiple when "Logger" is zap.
 	LogOutputs []string `json:"log-outputs"`
 
-	// zapLoggerBuilder is used to build the zap logger.
-	zapLoggerBuilder func(*Config) error
+	// ZapLoggerBuilder is used to build the zap logger.
+	ZapLoggerBuilder func(*Config) error
 
 	// logger logs server-side operations. The default is nil,
 	// and "setupLogging" must be called before starting server.

embed/config_logging.go

@@ -181,8 +181,8 @@ func (cfg *Config) setupLogging() error {
 				// TODO: remove "Debug" check in v3.5
 				grpc.EnableTracing = true
 			}
-			if cfg.zapLoggerBuilder == nil {
-				cfg.zapLoggerBuilder = func(c *Config) error {
+			if cfg.ZapLoggerBuilder == nil {
+				cfg.ZapLoggerBuilder = func(c *Config) error {
 					var err error
 					c.logger, err = copied.Build()
 					if err != nil {
@@ -235,8 +235,8 @@ func (cfg *Config) setupLogging() error {
 				syncer,
 				lvl,
 			)
-			if cfg.zapLoggerBuilder == nil {
-				cfg.zapLoggerBuilder = func(c *Config) error {
+			if cfg.ZapLoggerBuilder == nil {
+				cfg.ZapLoggerBuilder = func(c *Config) error {
 					c.logger = zap.New(cr, zap.AddCaller(), zap.ErrorOutput(syncer))
 					c.loggerMu.Lock()
 					defer c.loggerMu.Unlock()
@@ -252,7 +252,7 @@ func (cfg *Config) setupLogging() error {
 			}
 		}
 
-		err := cfg.zapLoggerBuilder(cfg)
+		err := cfg.ZapLoggerBuilder(cfg)
 		if err != nil {
 			return err
 		}

etcdctl/README.md

@@ -16,7 +16,7 @@ ETCDCTL_CERT=/tmp/cert.pem
 ETCDCTL_KEY=/tmp/key.pem
 ```
 
-Prefix flag strings with `ETCDCTL_`, convert all letters to upper-case, and replace dash(`-`) with underscore(`_`).
+Prefix flag strings with `ETCDCTL_`, convert all letters to upper-case, and replace dash(`-`) with underscore(`_`). Note that the environment variables with the prefix `ETCDCTL_` can only be used with the etcdctl global flags. Also, the environment variable `ETCDCTL_API` is a special case variable for etcdctl internal use only.
 
 ## Key-value commands
 

etcdctl/ctlv3/command/member_command.go

@@ -158,12 +158,14 @@ func memberAddCommandFunc(cmd *cobra.Command, args []string) {
 	if _, ok := (display).(*simplePrinter); ok {
 		ctx, cancel = commandCtx(cmd)
 		listResp, err := cli.MemberList(ctx)
-		// get latest member list; if there's failover new member might have outdated list
+		// make sure the member who served member list request has the latest member list.
+		syncedMemberSet := make(map[uint64]struct{})
+		syncedMemberSet[resp.Header.MemberId] = struct{}{} // the member who served member add is guaranteed to have the latest member list.
 		for {
 			if err != nil {
 				ExitWithError(ExitError, err)
 			}
-			if listResp.Header.MemberId == resp.Header.MemberId {
+			if _, ok := syncedMemberSet[listResp.Header.MemberId]; ok {
 				break
 			}
 			// quorum get to sync cluster list
@@ -171,7 +173,7 @@ func memberAddCommandFunc(cmd *cobra.Command, args []string) {
 			if gerr != nil {
 				ExitWithError(ExitError, err)
 			}
-			resp.Header.MemberId = gresp.Header.MemberId
+			syncedMemberSet[gresp.Header.MemberId] = struct{}{}
 			listResp, err = cli.MemberList(ctx)
 		}
 		cancel()

etcdmain/config.go

@@ -240,7 +240,7 @@ func newConfig() *config {
 	fs.BoolVar(&cfg.ec.EnablePprof, "enable-pprof", false, "Enable runtime profiling data via HTTP server. Address is at client URL + \"/debug/pprof/\"")
 
 	// additional metrics
-	fs.StringVar(&cfg.ec.Metrics, "metrics", cfg.ec.Metrics, "Set level of detail for exported metrics, specify 'extensive' to include histogram metrics")
+	fs.StringVar(&cfg.ec.Metrics, "metrics", cfg.ec.Metrics, "Set level of detail for exported metrics, specify 'extensive' to include server side grpc histogram metrics")
 
 	// auth
 	fs.StringVar(&cfg.ec.AuthToken, "auth-token", cfg.ec.AuthToken, "Specify auth token specific options.")

etcdmain/help.go

@@ -167,7 +167,7 @@ Profiling and Monitoring:
   --enable-pprof 'false'
     Enable runtime profiling data via HTTP server. Address is at client URL + "/debug/pprof/"
   --metrics 'basic'
-    Set level of detail for exported metrics, specify 'extensive' to include histogram metrics.
+    Set level of detail for exported metrics, specify 'extensive' to include server side grpc histogram metrics.
   --listen-metrics-urls ''
     List of URLs to listen on for the metrics and health endpoints.
 

etcdserver/api/membership/cluster.go

@@ -759,16 +759,21 @@ func ValidateClusterAndAssignIDs(lg *zap.Logger, local *RaftCluster, existing *R
 	if len(ems) != len(lms) {
 		return fmt.Errorf("member count is unequal")
 	}
-	sort.Sort(MembersByPeerURLs(ems))
-	sort.Sort(MembersByPeerURLs(lms))
 
 	ctx, cancel := context.WithTimeout(context.TODO(), 30*time.Second)
 	defer cancel()
 	for i := range ems {
-		if ok, err := netutil.URLStringsEqual(ctx, lg, ems[i].PeerURLs, lms[i].PeerURLs); !ok {
-			return fmt.Errorf("unmatched member while checking PeerURLs (%v)", err)
+		var err error
+		ok := false
+		for j := range lms {
+			if ok, err = netutil.URLStringsEqual(ctx, lg, ems[i].PeerURLs, lms[j].PeerURLs); ok {
+				lms[j].ID = ems[i].ID
+				break
+			}
+		}
+		if !ok {
+			return fmt.Errorf("PeerURLs: no match found for existing member (%v, %v), last resolver error (%v)", ems[i].ID, ems[i].PeerURLs, err)
 		}
-		lms[i].ID = ems[i].ID
 	}
 	local.members = make(map[types.ID]*Member)
 	for _, m := range lms {

etcdserver/apply.go

@@ -26,6 +26,7 @@ import (
 	"go.etcd.io/etcd/lease"
 	"go.etcd.io/etcd/mvcc"
 	"go.etcd.io/etcd/mvcc/mvccpb"
+	"go.etcd.io/etcd/pkg/traceutil"
 	"go.etcd.io/etcd/pkg/types"
 
 	"github.com/gogo/protobuf/proto"
@@ -43,17 +44,18 @@ type applyResult struct {
 	// to being logically reflected by the node. Currently only used for
 	// Compaction requests.
 	physc <-chan struct{}
+	trace *traceutil.Trace
 }
 
 // applierV3 is the interface for processing V3 raft messages
 type applierV3 interface {
 	Apply(r *pb.InternalRaftRequest) *applyResult
 
-	Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, error)
-	Range(txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error)
+	Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error)
+	Range(ctx context.Context, txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error)
 	DeleteRange(txn mvcc.TxnWrite, dr *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error)
 	Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error)
-	Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, error)
+	Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, *traceutil.Trace, error)
 
 	LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error)
 	LeaseRevoke(lc *pb.LeaseRevokeRequest) (*pb.LeaseRevokeResponse, error)
@@ -119,15 +121,15 @@ func (a *applierV3backend) Apply(r *pb.InternalRaftRequest) *applyResult {
 	// call into a.s.applyV3.F instead of a.F so upper appliers can check individual calls
 	switch {
 	case r.Range != nil:
-		ar.resp, ar.err = a.s.applyV3.Range(nil, r.Range)
+		ar.resp, ar.err = a.s.applyV3.Range(context.TODO(), nil, r.Range)
 	case r.Put != nil:
-		ar.resp, ar.err = a.s.applyV3.Put(nil, r.Put)
+		ar.resp, ar.trace, ar.err = a.s.applyV3.Put(nil, r.Put)
 	case r.DeleteRange != nil:
 		ar.resp, ar.err = a.s.applyV3.DeleteRange(nil, r.DeleteRange)
 	case r.Txn != nil:
 		ar.resp, ar.err = a.s.applyV3.Txn(r.Txn)
 	case r.Compaction != nil:
-		ar.resp, ar.physc, ar.err = a.s.applyV3.Compaction(r.Compaction)
+		ar.resp, ar.physc, ar.trace, ar.err = a.s.applyV3.Compaction(r.Compaction)
 	case r.LeaseGrant != nil:
 		ar.resp, ar.err = a.s.applyV3.LeaseGrant(r.LeaseGrant)
 	case r.LeaseRevoke != nil:
@@ -174,32 +176,39 @@ func (a *applierV3backend) Apply(r *pb.InternalRaftRequest) *applyResult {
 	return ar
 }
 
-func (a *applierV3backend) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (resp *pb.PutResponse, err error) {
+func (a *applierV3backend) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (resp *pb.PutResponse, trace *traceutil.Trace, err error) {
 	resp = &pb.PutResponse{}
 	resp.Header = &pb.ResponseHeader{}
-
+	trace = traceutil.New("put",
+		a.s.getLogger(),
+		traceutil.Field{Key: "key", Value: string(p.Key)},
+		traceutil.Field{Key: "req_size", Value: proto.Size(p)},
+	)
 	val, leaseID := p.Value, lease.LeaseID(p.Lease)
 	if txn == nil {
 		if leaseID != lease.NoLease {
 			if l := a.s.lessor.Lookup(leaseID); l == nil {
-				return nil, lease.ErrLeaseNotFound
+				return nil, nil, lease.ErrLeaseNotFound
 			}
 		}
-		txn = a.s.KV().Write()
+		txn = a.s.KV().Write(trace)
 		defer txn.End()
 	}
 
 	var rr *mvcc.RangeResult
 	if p.IgnoreValue || p.IgnoreLease || p.PrevKv {
+		trace.DisableStep()
 		rr, err = txn.Range(p.Key, nil, mvcc.RangeOptions{})
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
+		trace.EnableStep()
+		trace.Step("get previous kv pair")
 	}
 	if p.IgnoreValue || p.IgnoreLease {
 		if rr == nil || len(rr.KVs) == 0 {
 			// ignore_{lease,value} flag expects previous key-value pair
-			return nil, ErrKeyNotFound
+			return nil, nil, ErrKeyNotFound
 		}
 	}
 	if p.IgnoreValue {
@@ -215,7 +224,8 @@ func (a *applierV3backend) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (resp *pb.Pu
 	}
 
 	resp.Header.Revision = txn.Put(p.Key, val, leaseID)
-	return resp, nil
+	trace.AddField(traceutil.Field{Key: "response_revision", Value: resp.Header.Revision})
+	return resp, trace, nil
 }
 
 func (a *applierV3backend) DeleteRange(txn mvcc.TxnWrite, dr *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) {
@@ -224,7 +234,7 @@ func (a *applierV3backend) DeleteRange(txn mvcc.TxnWrite, dr *pb.DeleteRangeRequ
 	end := mkGteRange(dr.RangeEnd)
 
 	if txn == nil {
-		txn = a.s.kv.Write()
+		txn = a.s.kv.Write(traceutil.TODO())
 		defer txn.End()
 	}
 
@@ -245,12 +255,14 @@ func (a *applierV3backend) DeleteRange(txn mvcc.TxnWrite, dr *pb.DeleteRangeRequ
 	return resp, nil
 }
 
-func (a *applierV3backend) Range(txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) {
+func (a *applierV3backend) Range(ctx context.Context, txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) {
+	trace := traceutil.Get(ctx)
+
 	resp := &pb.RangeResponse{}
 	resp.Header = &pb.ResponseHeader{}
 
 	if txn == nil {
-		txn = a.s.kv.Read()
+		txn = a.s.kv.Read(trace)
 		defer txn.End()
 	}
 
@@ -327,7 +339,7 @@ func (a *applierV3backend) Range(txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.Rang
 		rr.KVs = rr.KVs[:r.Limit]
 		resp.More = true
 	}
-
+	trace.Step("filter and sort the key-value pairs")
 	resp.Header.Revision = rr.Rev
 	resp.Count = int64(rr.Count)
 	resp.Kvs = make([]*mvccpb.KeyValue, len(rr.KVs))
@@ -337,12 +349,13 @@ func (a *applierV3backend) Range(txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.Rang
 		}
 		resp.Kvs[i] = &rr.KVs[i]
 	}
+	trace.Step("assemble the response")
 	return resp, nil
 }
 
 func (a *applierV3backend) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) {
 	isWrite := !isTxnReadonly(rt)
-	txn := mvcc.NewReadOnlyTxnWrite(a.s.KV().Read())
+	txn := mvcc.NewReadOnlyTxnWrite(a.s.KV().Read(traceutil.TODO()))
 
 	txnPath := compareToPath(txn, rt)
 	if isWrite {
@@ -364,7 +377,7 @@ func (a *applierV3backend) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) {
 	// be the revision of the write txn.
 	if isWrite {
 		txn.End()
-		txn = a.s.KV().Write()
+		txn = a.s.KV().Write(traceutil.TODO())
 	}
 	a.applyTxn(txn, rt, txnPath, txnResp)
 	rev := txn.Rev()
@@ -516,7 +529,7 @@ func (a *applierV3backend) applyTxn(txn mvcc.TxnWrite, rt *pb.TxnRequest, txnPat
 		respi := tresp.Responses[i].Response
 		switch tv := req.Request.(type) {
 		case *pb.RequestOp_RequestRange:
-			resp, err := a.Range(txn, tv.RequestRange)
+			resp, err := a.Range(context.TODO(), txn, tv.RequestRange)
 			if err != nil {
 				if lg != nil {
 					lg.Panic("unexpected error during txn", zap.Error(err))
@@ -526,7 +539,7 @@ func (a *applierV3backend) applyTxn(txn mvcc.TxnWrite, rt *pb.TxnRequest, txnPat
 			}
 			respi.(*pb.ResponseOp_ResponseRange).ResponseRange = resp
 		case *pb.RequestOp_RequestPut:
-			resp, err := a.Put(txn, tv.RequestPut)
+			resp, _, err := a.Put(txn, tv.RequestPut)
 			if err != nil {
 				if lg != nil {
 					lg.Panic("unexpected error during txn", zap.Error(err))
@@ -557,17 +570,22 @@ func (a *applierV3backend) applyTxn(txn mvcc.TxnWrite, rt *pb.TxnRequest, txnPat
 	return txns
 }
 
-func (a *applierV3backend) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, error) {
+func (a *applierV3backend) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, *traceutil.Trace, error) {
 	resp := &pb.CompactionResponse{}
 	resp.Header = &pb.ResponseHeader{}
-	ch, err := a.s.KV().Compact(compaction.Revision)
+	trace := traceutil.New("compact",
+		a.s.getLogger(),
+		traceutil.Field{Key: "revision", Value: compaction.Revision},
+	)
+
+	ch, err := a.s.KV().Compact(trace, compaction.Revision)
 	if err != nil {
-		return nil, ch, err
+		return nil, ch, nil, err
 	}
 	// get the current revision. which key to get is not important.
 	rr, _ := a.s.KV().Range([]byte("compaction"), nil, mvcc.RangeOptions{})
 	resp.Header.Revision = rr.Rev
-	return resp, ch, err
+	return resp, ch, trace, err
 }
 
 func (a *applierV3backend) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) {
@@ -674,8 +692,8 @@ type applierV3Capped struct {
 // with Puts so that the number of keys in the store is capped.
 func newApplierV3Capped(base applierV3) applierV3 { return &applierV3Capped{applierV3: base} }
 
-func (a *applierV3Capped) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, error) {
-	return nil, ErrNoSpace
+func (a *applierV3Capped) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) {
+	return nil, nil, ErrNoSpace
 }
 
 func (a *applierV3Capped) Txn(r *pb.TxnRequest) (*pb.TxnResponse, error) {
@@ -824,13 +842,13 @@ func newQuotaApplierV3(s *EtcdServer, app applierV3) applierV3 {
 	return &quotaApplierV3{app, NewBackendQuota(s, "v3-applier")}
 }
 
-func (a *quotaApplierV3) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, error) {
+func (a *quotaApplierV3) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) {
 	ok := a.q.Available(p)
-	resp, err := a.applierV3.Put(txn, p)
+	resp, trace, err := a.applierV3.Put(txn, p)
 	if err == nil && !ok {
 		err = ErrNoSpace
 	}
-	return resp, err
+	return resp, trace, err
 }
 
 func (a *quotaApplierV3) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) {

etcdserver/apply_auth.go

@@ -15,12 +15,14 @@
 package etcdserver
 
 import (
+	"context"
 	"sync"
 
 	"go.etcd.io/etcd/auth"
 	pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
 	"go.etcd.io/etcd/lease"
 	"go.etcd.io/etcd/mvcc"
+	"go.etcd.io/etcd/pkg/traceutil"
 )
 
 type authApplierV3 struct {
@@ -61,9 +63,9 @@ func (aa *authApplierV3) Apply(r *pb.InternalRaftRequest) *applyResult {
 	return ret
 }
 
-func (aa *authApplierV3) Put(txn mvcc.TxnWrite, r *pb.PutRequest) (*pb.PutResponse, error) {
+func (aa *authApplierV3) Put(txn mvcc.TxnWrite, r *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) {
 	if err := aa.as.IsPutPermitted(&aa.authInfo, r.Key); err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if err := aa.checkLeasePuts(lease.LeaseID(r.Lease)); err != nil {
@@ -71,23 +73,23 @@ func (aa *authApplierV3) Put(txn mvcc.TxnWrite, r *pb.PutRequest) (*pb.PutRespon
 		// be written by this user. It means the user cannot revoke the
 		// lease so attaching the lease to the newly written key should
 		// be forbidden.
-		return nil, err
+		return nil, nil, err
 	}
 
 	if r.PrevKv {
 		err := aa.as.IsRangePermitted(&aa.authInfo, r.Key, nil)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 	}
 	return aa.applierV3.Put(txn, r)
 }
 
-func (aa *authApplierV3) Range(txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) {
+func (aa *authApplierV3) Range(ctx context.Context, txn mvcc.TxnRead, r *pb.RangeRequest) (*pb.RangeResponse, error) {
 	if err := aa.as.IsRangePermitted(&aa.authInfo, r.Key, r.RangeEnd); err != nil {
 		return nil, err
 	}
-	return aa.applierV3.Range(txn, r)
+	return aa.applierV3.Range(ctx, txn, r)
 }
 
 func (aa *authApplierV3) DeleteRange(txn mvcc.TxnWrite, r *pb.DeleteRangeRequest) (*pb.DeleteRangeResponse, error) {

etcdserver/corrupt.go

@@ -23,6 +23,7 @@ import (
 	"go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
 	pb "go.etcd.io/etcd/etcdserver/etcdserverpb"
 	"go.etcd.io/etcd/mvcc"
+	"go.etcd.io/etcd/pkg/traceutil"
 	"go.etcd.io/etcd/pkg/types"
 
 	"go.uber.org/zap"
@@ -382,11 +383,11 @@ type applierV3Corrupt struct {
 
 func newApplierV3Corrupt(a applierV3) *applierV3Corrupt { return &applierV3Corrupt{a} }
 
-func (a *applierV3Corrupt) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, error) {
-	return nil, ErrCorrupt
+func (a *applierV3Corrupt) Put(txn mvcc.TxnWrite, p *pb.PutRequest) (*pb.PutResponse, *traceutil.Trace, error) {
+	return nil, nil, ErrCorrupt
 }
 
-func (a *applierV3Corrupt) Range(txn mvcc.TxnRead, p *pb.RangeRequest) (*pb.RangeResponse, error) {
+func (a *applierV3Corrupt) Range(ctx context.Context, txn mvcc.TxnRead, p *pb.RangeRequest) (*pb.RangeResponse, error) {
 	return nil, ErrCorrupt
 }
 
@@ -398,8 +399,8 @@ func (a *applierV3Corrupt) Txn(rt *pb.TxnRequest) (*pb.TxnResponse, error) {
 	return nil, ErrCorrupt
 }
 
-func (a *applierV3Corrupt) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, error) {
-	return nil, nil, ErrCorrupt
+func (a *applierV3Corrupt) Compaction(compaction *pb.CompactionRequest) (*pb.CompactionResponse, <-chan struct{}, *traceutil.Trace, error) {
+	return nil, nil, nil, ErrCorrupt
 }
 
 func (a *applierV3Corrupt) LeaseGrant(lc *pb.LeaseGrantRequest) (*pb.LeaseGrantResponse, error) {

etcdserver/server.go

@@ -50,6 +50,7 @@ import (
 	"go.etcd.io/etcd/pkg/pbutil"
 	"go.etcd.io/etcd/pkg/runtime"
 	"go.etcd.io/etcd/pkg/schedule"
+	"go.etcd.io/etcd/pkg/traceutil"
 	"go.etcd.io/etcd/pkg/types"
 	"go.etcd.io/etcd/pkg/wait"
 	"go.etcd.io/etcd/raft"
@@ -1178,7 +1179,7 @@ func (s *EtcdServer) applySnapshot(ep *etcdProgress, apply *apply) {
 			plog.Info("recovering lessor...")
 		}
 
-		s.lessor.Recover(newbe, func() lease.TxnDelete { return s.kv.Write() })
+		s.lessor.Recover(newbe, func() lease.TxnDelete { return s.kv.Write(traceutil.TODO()) })
 
 		if lg != nil {
 			lg.Info("restored lease store")

etcdserver/v3_server.go

@@ -26,6 +26,7 @@ import (
 	"go.etcd.io/etcd/lease"
 	"go.etcd.io/etcd/lease/leasehttp"
 	"go.etcd.io/etcd/mvcc"
+	"go.etcd.io/etcd/pkg/traceutil"
 	"go.etcd.io/etcd/raft"
 
 	"github.com/gogo/protobuf/proto"
@@ -38,6 +39,7 @@ const (
 	// However, if the committed entries are very heavy to apply, the gap might grow.
 	// We should stop accepting new proposals if the gap growing to a certain point.
 	maxGapBetweenApplyAndCommitIndex = 5000
+	traceThreshold                   = 100 * time.Millisecond
 )
 
 type RaftKV interface {
@@ -85,14 +87,29 @@ type Authenticator interface {
 }
 
 func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeResponse, error) {
+	trace := traceutil.New("range",
+		s.getLogger(),
+		traceutil.Field{Key: "range_begin", Value: string(r.Key)},
+		traceutil.Field{Key: "range_end", Value: string(r.RangeEnd)},
+	)
+	ctx = context.WithValue(ctx, traceutil.TraceKey, trace)
+
 	var resp *pb.RangeResponse
 	var err error
 	defer func(start time.Time) {
 		warnOfExpensiveReadOnlyRangeRequest(s.getLogger(), start, r, resp, err)
+		if resp != nil {
+			trace.AddField(
+				traceutil.Field{Key: "response_count", Value: len(resp.Kvs)},
+				traceutil.Field{Key: "response_revision", Value: resp.Header.Revision},
+			)
+		}
+		trace.LogIfLong(traceThreshold)
 	}(time.Now())
 
 	if !r.Serializable {
 		err = s.linearizableReadNotify(ctx)
+		trace.Step("agreement among raft nodes before linearized reading")
 		if err != nil {
 			return nil, err
 		}
@@ -101,7 +118,7 @@ func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeRe
 		return s.authStore.IsRangePermitted(ai, r.Key, r.RangeEnd)
 	}
 
-	get := func() { resp, err = s.applyV3Base.Range(nil, r) }
+	get := func() { resp, err = s.applyV3Base.Range(ctx, nil, r) }
 	if serr := s.doSerialize(ctx, chk, get); serr != nil {
 		err = serr
 		return nil, err
@@ -110,6 +127,7 @@ func (s *EtcdServer) Range(ctx context.Context, r *pb.RangeRequest) (*pb.RangeRe
 }
 
 func (s *EtcdServer) Put(ctx context.Context, r *pb.PutRequest) (*pb.PutResponse, error) {
+	ctx = context.WithValue(ctx, traceutil.StartTimeKey, time.Now())
 	resp, err := s.raftRequest(ctx, pb.InternalRaftRequest{Put: r})
 	if err != nil {
 		return nil, err
@@ -186,7 +204,18 @@ func isTxnReadonly(r *pb.TxnRequest) bool {
 }
 
 func (s *EtcdServer) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.CompactionResponse, error) {
+	startTime := time.Now()
 	result, err := s.processInternalRaftRequestOnce(ctx, pb.InternalRaftRequest{Compaction: r})
+	trace := traceutil.TODO()
+	if result != nil && result.trace != nil {
+		trace = result.trace
+		defer func() {
+			trace.LogIfLong(traceThreshold)
+		}()
+		applyStart := result.trace.GetStartTime()
+		result.trace.SetStartTime(startTime)
+		trace.InsertStep(0, applyStart, "process raft request")
+	}
 	if r.Physical && result != nil && result.physc != nil {
 		<-result.physc
 		// The compaction is done deleting keys; the hash is now settled
@@ -195,6 +224,7 @@ func (s *EtcdServer) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.
 		// if the compaction resumes. Force the finished compaction to
 		// commit so it won't resume following a crash.
 		s.be.ForceCommit()
+		trace.Step("physically apply compaction")
 	}
 	if err != nil {
 		return nil, err
@@ -210,6 +240,7 @@ func (s *EtcdServer) Compact(ctx context.Context, r *pb.CompactionRequest) (*pb.
 		resp.Header = &pb.ResponseHeader{}
 	}
 	resp.Header.Revision = s.kv.Rev()
+	trace.AddField(traceutil.Field{Key: "response_revision", Value: resp.Header.Revision})
 	return resp, nil
 }
 
@@ -533,6 +564,15 @@ func (s *EtcdServer) raftRequestOnce(ctx context.Context, r pb.InternalRaftReque
 	if result.err != nil {
 		return nil, result.err
 	}
+	if startTime, ok := ctx.Value(traceutil.StartTimeKey).(time.Time); ok && result.trace != nil {
+		applyStart := result.trace.GetStartTime()
+		// The trace object is created in apply. Here reset the start time to trace
+		// the raft request time by the difference between the request start time
+		// and apply start time
+		result.trace.SetStartTime(startTime)
+		result.trace.InsertStep(0, applyStart, "process raft request")
+		result.trace.LogIfLong(traceThreshold)
+	}
 	return result.resp, nil
 }
 
@@ -547,6 +587,7 @@ func (s *EtcdServer) raftRequest(ctx context.Context, r pb.InternalRaftRequest)
 
 // doSerialize handles the auth logic, with permissions checked by "chk", for a serialized request "get". Returns a non-nil error on authentication failure.
 func (s *EtcdServer) doSerialize(ctx context.Context, chk func(*auth.AuthInfo) error, get func()) error {
+	trace := traceutil.Get(ctx)
 	ai, err := s.AuthInfoFromCtx(ctx)
 	if err != nil {
 		return err
@@ -558,6 +599,7 @@ func (s *EtcdServer) doSerialize(ctx context.Context, chk func(*auth.AuthInfo) e
 	if err = chk(ai); err != nil {
 		return err
 	}
+	trace.Step("get authentication metadata")
 	// fetch response for serialized request
 	get()
 	// check for stale token revision in case the auth store was updated while

functional/scripts/docker-local-agent.sh

@@ -13,7 +13,7 @@ if ! [[ "${0}" =~ "scripts/docker-local-agent.sh" ]]; then
 fi
 
 if [[ -z "${GO_VERSION}" ]]; then
-  GO_VERSION=1.13
+  GO_VERSION=1.13.1
 fi
 echo "Running with GO_VERSION:" ${GO_VERSION}
 

functional/scripts/docker-local-tester.sh

@@ -6,7 +6,7 @@ if ! [[ "${0}" =~ "scripts/docker-local-tester.sh" ]]; then
 fi
 
 if [[ -z "${GO_VERSION}" ]]; then
-  GO_VERSION=1.13
+  GO_VERSION=1.13.1
 fi
 echo "Running with GO_VERSION:" ${GO_VERSION}
 

functional/tester/cluster_test.go

@@ -64,7 +64,6 @@ func Test_read(t *testing.T) {
 					InitialCorruptCheck: true,
 					Logger:              "zap",
 					LogOutputs:          []string{"/tmp/etcd-functional-1/etcd.log"},
-					Debug:               true,
 				},
 				ClientCertData:      "",
 				ClientCertPath:      "",
@@ -117,7 +116,6 @@ func Test_read(t *testing.T) {
 					InitialCorruptCheck: true,
 					Logger:              "zap",
 					LogOutputs:          []string{"/tmp/etcd-functional-2/etcd.log"},
-					Debug:               true,
 				},
 				ClientCertData:      "",
 				ClientCertPath:      "",
@@ -170,7 +168,6 @@ func Test_read(t *testing.T) {
 					InitialCorruptCheck: true,
 					Logger:              "zap",
 					LogOutputs:          []string{"/tmp/etcd-functional-3/etcd.log"},
-					Debug:               true,
 				},
 				ClientCertData:      "",
 				ClientCertPath:      "",

go.mod

@@ -43,7 +43,7 @@ require (
 	golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7
 	golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 // indirect
 	golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2
-	google.golang.org/grpc v1.23.0
+	google.golang.org/grpc v1.24.0
 	gopkg.in/cheggaaa/pb.v1 v1.0.25
 	gopkg.in/yaml.v2 v2.2.2
 	sigs.k8s.io/yaml v1.1.0

go.sum

@@ -181,8 +181,8 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A=
-google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.24.0 h1:vb/1TCsVn3DcJlQ0Gs1yB1pKI6Do2/QNwxdKqmc/b0s=
+google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

hack/patch/README.md

@@ -7,14 +7,14 @@ Handles cherry-picks of PR(s) from etcd master to a stable etcd release branch a
 Set the `UPSTREAM_REMOTE` and `FORK_REMOTE` environment variables.
 `UPSTREAM_REMOTE` should be set to git remote name of `github.com/etcd-io/etcd`,
 and `FORK_REMOTE` should be set to the git remote name of the forked etcd
-repo (`github.com/${github-username}/etcd`). Use `git remotes -v` to
+repo (`github.com/${github-username}/etcd`). Use `git remote -v` to
 look up the git remote names. If etcd has not been forked, create
 one on github.com and register it locally with `git remote add ...`.
 
 
 ```
-export UPSTREAM_REMOTE=origin
-export FORK_REMOTE=${github-username}
+export UPSTREAM_REMOTE=upstream
+export FORK_REMOTE=origin
 export GITHUB_USER=${github-username}
 ```
 

integration/fixtures/ca.crt

@@ -1,22 +1,22 @@
 -----BEGIN CERTIFICATE-----
-MIIDrjCCApagAwIBAgIUM24Z44NdsHtDQisQRIH+mmhXLHYwDQYJKoZIhvcNAQEL
+MIIDrjCCApagAwIBAgIUOl7DCgSvqQKhiihYrZDiBKNpQX4wDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTAxMjExNDQwMDBaFw0yOTAxMTgxNDQw
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
 MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
 BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
 ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
-AoIBAQCsWG1qafiCwfmKEltvpmslNqOlWgMp9H7VIP7cExbhsW4P1L4Jlfcz7rFH
-2MFpwktbxppoYI/4umTj5r7dx/K6mttUBtiLY5VwSCo/asZvLaOLFN2QP4cwkpLI
-lFDy4Pez2Uu+NnmMF6SLq+M6mOaHSbURNvphP1zWX9SLKo1OV8GT6r/oHYmcR+xy
-skWd/+6B73S0pbG/d3ME/WoovZtOXqaZtJn8YIBXE6LGd4NBkSK3Jg9c4QzlErTM
-j6ItTs7t9aPjXd2kiq8IY6UN2TrLwssWkGM4Oop+mlp5zcKIDLhDrfsRga5hxx2Z
-i0coNWBKNjvVaCO0L7Qn1nIHA1KtAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
-BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTdZTAnocY85lHhyR/A7lJkp3t2mjAN
-BgkqhkiG9w0BAQsFAAOCAQEABCTckIi6zoE7uSy71uNO93RC/Pcb+YzmRSNQzl60
-ngUlUrd+18bjp3O9u8jQ8ikhWT3jfn5e4I1nqLKFqKP6xyMPwk2ZJXF3WeBvtuHW
-BonDscbYwMpL6RDgcUU1+2ZtZYlo+NZkeXQdTO0Pa8qoo/EtNXb+Bg1FFqnrLrVI
-EhY3Bd5+jvC0WkjJFMFeOUkZDmtKLX24P/901ZP+6HN2bA+MIBKmIDKbctP54J73
-tncuOOFBfyWkckIMISM4D+Mi9Ezju2Hq4thV7XJeyWTRiXG8+LhVRWJaz7St1FIw
-ViEST3A84CBLjiPyGqzqQCtr+HNhr7su+Tmcq550xU11Ug==
+AoIBAQDBNhwKD8oqOwNSDMZR+K6l6ocyXZzZPAIbv7co34xtjt25c8PPKz8FiBSU
+M4YeZpzsSp7n7WSSSzVWqFTRBZzvjIrBzLu4CfxMKuUrQX1/BPYgbSxQO+5YKPzO
+yaBMhIAEtW+WYsaa6PpWyL65L4giKpVoLS/UFTEBsf+lO6pwFpX2EJnIylLbpwEd
+pAXIgVFsodHlP9Zc2tR1TqYetmJ6/A/p5sSZpgLy1y2+Mg4VTMKvs2kNAoh/+lEu
+WPe204eMpkBXhukulOiJkVKNdhnCkLslt8ZaMWWqBvD9d94lXycMQ9wnGakPNc4W
+5VX3rbLOGOX7xK37BCsh5HGodIrZAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
+BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRlB76vjaZyFLrEUGm6DQfyjmN6PjAN
+BgkqhkiG9w0BAQsFAAOCAQEAD0cRNBQqOPNAUmKCH9xCr4TZFoE+P5aNePU39Jyp
+qpJ1HjKI93zBk9aN5udDGPFhm2/iaKx6DuABbxCz0LwNhLiKP6UbHV8F2fTJJ5bo
+crXvD0CEpor+Quh995lbq9bv29+zcDVw+Hw0QainBdHWkdw6RAgmbFnJxETDDz8z
+VQ0DET3T736oxpEZ4DKQlbzK5LSgZH2lyPEEvzci4QjTZf5X/nitdx7fAdMFFPQ0
+lI4l7nIuge5LTR0isEfWHx7Orx6l8dzkofG3fz5BjHCI4JInVlWq3MNNSybDI4pI
+GFxeuE/U8K6kIixT8qCAh6Naq9/xuxFkffLmMKfZXoYLCg==
 -----END CERTIFICATE-----

integration/fixtures/gencerts.sh

@@ -43,6 +43,15 @@ cfssl gencert \
 mv server-ip.pem server-ip.crt
 mv server-ip-key.pem server-ip.key.insecure
 
+# generate IPv6: [::1], CN: example.com certificates
+cfssl gencert \
+  --ca ./ca.crt \
+  --ca-key ./ca-key.pem \
+  --config ./gencert.json \
+  ./server-ca-csr-ipv6.json | cfssljson --bare ./server-ip
+mv server-ip.pem server-ipv6.crt
+mv server-ip-key.pem server-ipv6.key.insecure
+
 # generate DNS: localhost, IP: 127.0.0.1, CN: example2.com certificates
 cfssl gencert \
   --ca ./ca.crt \

integration/fixtures/server-ca-csr-ipv6.json

@@ -0,0 +1,19 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "example.com",
+  "hosts": [
+    "::1"
+  ]
+}

integration/fixtures/server-ecdsa.crt

@@ -1,20 +1,20 @@
 -----BEGIN CERTIFICATE-----
-MIIDRzCCAi+gAwIBAgIUMEJ5c+Tt0TyOcLL+dIUuE2nOuukwDQYJKoZIhvcNAQEL
+MIIDRzCCAi+gAwIBAgIUKgQJ/CMaFxc4JcwwGyiT/7KpedIwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTAxMjExNDQwMDBaFw0yOTAxMTgxNDQw
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
 BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
 ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wWTATBgcqhkjOPQIBBggqhkjO
-PQMBBwNCAAS/sa5Guqq+tML3vUYITeBQ7gURu5yJa0gKALVIpQ75AJXG9fp1dMD1
-+3M7HiT56p5omwDPqe8zFsCvPSm6TSEPo4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAd
+PQMBBwNCAARXbc8naiFZ3Y2LujrnDCScVNRks/TR+aXPmnuPGjDxbuHxSSbC8Q2z
+iTvCkgsIcsifmUIEQcI4v3Kbkj3qMF1so4GcMIGZMA4GA1UdDwEB/wQEAwIFoDAd
 BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNV
-HQ4EFgQUOErhH7Ot6qYob29cUsijrKhSGawwHwYDVR0jBBgwFoAU3WUwJ6HGPOZR
-4ckfwO5SZKd7dpowGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3
-DQEBCwUAA4IBAQBOjMqQ2AGHTOvHiG1eumKDaxGzXGb7znMcnpKYuz0OT97IoZSw
-EggwwUbqaK+9DotDcAWaqkReP18P3T9TgzZMfFDFctSKB5rM4EU2iPpAHdA6EEB8
-87HutlAeFphjsRlUMRLZ2YvTutR0jVeniEDTmTUB9crhGuUrCbg5H8jsVjvDKDut
-si3l6jsm598EWYa2P7ac5/MXQ5/Z9QCMogE/zOPzbnHNuAbf6ZdGFHNM6cgUgHvs
-C8L6hnuOCouFfcNDRK+7WjpIde18LNwLC0AwCKXbwFdWErRWJ8W978t6htdBYS9p
-cvvxQXBuMRmAykhjKaE+rvjdV3IJqQU8mGLY
+HQ4EFgQU3z1DifT82BfoU5DfMe08meeYmSUwHwYDVR0jBBgwFoAUZQe+r42mchS6
+xFBpug0H8o5jej4wGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMA0GCSqGSIb3
+DQEBCwUAA4IBAQAE3bhZcJuGrnMGMgebCFMuAXvoF9twYIHXpxNOg6u0HTIWOsMB
+njEJW/rfZFE/RAJ6JdOMNE2bq2LbJ8dUA25PX3uz6V4omm9B3EvEG9Hh3J+C77XQ
+P+ofiUd+j06SdewoxrmmQmjZZdotpFUQG3EEncs+v94jsamwGNLdq4yWDjFdmyuC
+hqzSkD48aGqP2Q93wfv8uIiCEmJS1vITTm2LxssCLfiYGortpCx32/DWme8nUlni
+1U/pRTx8Brx00dMeruTGjCCpwb8k453oNV6u0D1LsQ9y5DuyEwmZtBEHBN1kVPro
+yYW3/b1jcmZk8W9GXqcXy16LbWmpvJmTHPsj
 -----END CERTIFICATE-----

integration/fixtures/server-ecdsa.key.insecure

@@ -1,5 +1,5 @@
 -----BEGIN EC PRIVATE KEY-----
-MHcCAQEEINToOjKwxXFyCQHkiWoL55IPdPoYhm1TFmDylAUIhJWZoAoGCCqGSM49
-AwEHoUQDQgAEv7GuRrqqvrTC971GCE3gUO4FEbuciWtICgC1SKUO+QCVxvX6dXTA
-9ftzOx4k+eqeaJsAz6nvMxbArz0puk0hDw==
+MHcCAQEEIK3K2gimOw2P0pZ4soFAopriuORuqpRptllFXNRhCRV0oAoGCCqGSM49
+AwEHoUQDQgAEV23PJ2ohWd2Ni7o65wwknFTUZLP00fmlz5p7jxow8W7h8UkmwvEN
+s4k7wpILCHLIn5lCBEHCOL9ym5I96jBdbA==
 -----END EC PRIVATE KEY-----

integration/fixtures/server-ip.crt

@@ -1,24 +1,24 @@
 -----BEGIN CERTIFICATE-----
-MIIEBzCCAu+gAwIBAgIUGZReOLZEaMEZ2PfqR25XMrEdBIMwDQYJKoZIhvcNAQEL
+MIIEBzCCAu+gAwIBAgIUSvxuG1lgImYpnaK4sPaCiMAd0lgwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTAxMjExNDQwMDBaFw0yOTAxMTgxNDQw
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
 BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
 ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
-A4IBDwAwggEKAoIBAQDPXZtt7WyGbsRRzkKHTKqeqAGaFdVJpeLCiLpq15hGiOGl
-RJFFrQ/SxEH2y8CmoKLcY96uKYxzVWHpPStK0wa/3DMTE0sxhdWFixD/eRTNgA/o
-ovvSEPTX/ya//DfrgvrKNeSCG/E3hDXitVdiexeUiIB8DZHwdAg82Zg9eJ41ck+G
-WD9u//PwUqS8epqs15xXaHMQphjATnkLa/0mIjwo6JPddtGopBQRADaorjvpaoUu
-SzL7TQaHzCVuj47szr7BmNK1mwoHXJk7d+BlBJz6SiSGBLLq3h3SmoX9/yDLNU7t
-rCi9Yl55/ITJEfY56ZZ6L/BLm83b9r03lv1vYYr/AgMBAAGjgZEwgY4wDgYDVR0P
+A4IBDwAwggEKAoIBAQC7mJOiyqWfmNM5ptQZ22plotVfgoBf9fHTzMw/ap2Vl0/0
+4V3GEyYCdPt6V87GWzjBSO9GAmlISBQQybMieZTaTm8KKW2066iJDKseBCv9m4nS
+mHv0oDqp3SHsZQ2xHis4lbi7ws2thdqpmjw4Dv96SUiCJUjhcBX4kBMRcOGgk1RF
+ENIOInTSKlAiwNF1NSnhj8wMNw7mjw90jpAGAuPuuiQ7+AYHJBJqtT9mRikR8ppw
+isjEE6kslCCg2RC45AiF4LXNp7A7Xwm6P34XJ6T9PJUh/r3pa0xHRuI2zQLaW8Z/
+b6NYkUGMbHR7AY/+2JzOfnnnQcSB8EYC9bHadvHnAgMBAAGjgZEwgY4wDgYDVR0P
 AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
-Af8EAjAAMB0GA1UdDgQWBBRn6Ksd8Ra9fiY7fCZnmq2OylkQmTAfBgNVHSMEGDAW
-gBTdZTAnocY85lHhyR/A7lJkp3t2mjAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3
-DQEBCwUAA4IBAQCj+s26SaG99nC/OAHJtXQqxyqDfoKNwO6iwK4UGGzwlKAK9+a7
-8ObVOIyAbFtHUFjFJ6cIBMg+Tw+9++bRPyliOcrIDiv9ytNEzMVIQq07oj0Kx7Qw
-sSYcIeFRF439ftiHC0LAULFEV/hDBveuTVfdt3t5RnEzp8PiTjXvhSpgFOhkuln8
-n3NK4UoolN6gJ/sSCP91Oka90l4xagPYK37mksYfzbTBNmPB88rMgioee5ZU1nCG
-09fHiNWg/U8c0R6Iflpjy3lsUlnst3+VZp7HZ1mO+hBp7p0lduAdJqZs0ev9gjhi
-odfA+/2O8LPUTTXz6lpdnou2kXl0B4I0jG2v
+Af8EAjAAMB0GA1UdDgQWBBSPaFA2Jh7s/IJN/Yw/QFFR4pO3nDAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjAPBgNVHREECDAGhwR/AAABMA0GCSqGSIb3
+DQEBCwUAA4IBAQAO2EnUXDlZAzOJLmkzQQF/d88PjvzspFtBfj/jCGzK6bpjeZwq
+oM1fQOkjuFeNvVLA3WHVT0XEpZEM8lwAr/YwnBWMFlNd3Vb2Cho5VaQq0nVfhYoB
+tpzoWcf0Qx4cALesQZ3y2EnXePpzky1R4MfHqulYrmZKSBQsERob/7YgSBk+ucV9
+OHLzYxm4OvYvDoR54REq+vgZ3ohoDmBrNNv9OmUHLIrUi+nBpBgnww85Dc7cKB27
+EEKxqIfCNTeHSemvzfK/1M6manQX6eyGe48nOwQMV/ocfY6SeA7RABT0l/UsbeMp
+g/b2RU+liZ3e8FziW4/1VTt1pmFAN/2hnb0v
 -----END CERTIFICATE-----

integration/fixtures/server-ip.key.insecure

@@ -1,27 +1,27 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEAz12bbe1shm7EUc5Ch0yqnqgBmhXVSaXiwoi6ateYRojhpUSR
-Ra0P0sRB9svApqCi3GPerimMc1Vh6T0rStMGv9wzExNLMYXVhYsQ/3kUzYAP6KL7
-0hD01/8mv/w364L6yjXkghvxN4Q14rVXYnsXlIiAfA2R8HQIPNmYPXieNXJPhlg/
-bv/z8FKkvHqarNecV2hzEKYYwE55C2v9JiI8KOiT3XbRqKQUEQA2qK476WqFLksy
-+00Gh8wlbo+O7M6+wZjStZsKB1yZO3fgZQSc+kokhgSy6t4d0pqF/f8gyzVO7awo
-vWJeefyEyRH2OemWei/wS5vN2/a9N5b9b2GK/wIDAQABAoIBABMnzIHdGtdYSB5e
-dVrWRDSfxHYdajSBdG8P/lh8Tf7GCkIgEWNkVz/bDVTlAYji8eh1+U4RXH4S9xZ6
-phMlZ0w15Snv8FREzrKlZA6Vesx49f9Bfw2qr1N8qHG3tNq2oMApNlCmkCPWvLuS
-kN3yDP2VlnjfMAoMTe6BE0UqbUL2f58h2LBo8x04FYRye5VtroO+lM/9cHUB0xAJ
-Lsp7hiu5wWsORyn6pkpC5B/Fw2sqOdEKtImcs9CkimkbZAfQXq97Rd/jAkAoVaEB
-j4lKri9bKwiTH7Xi9LyH8Ix445lqC3uT7728yltTiyPHME0O6RsP5AdfZRmGCrEg
-cMuplFECgYEA1gUmENbRx6zQdQ0CiwgRloRzceLd3l0lJCxaohqKfX28/keBjMJY
-BB/EM+Qv+aBwrOshSAAW6EttBdqKMs3/EIngCpvMQfAKN7flECqfPZpGJX5BV+E/
-51E4wjwCrxpmkYlQ8qj7lt8wcNPU55Tw9bfKelEgQ+dfyD6TpWV9pnsCgYEA+ApP
-XfGsSkeaFsx6LJIy3ptYxrcd4Xus7ELv6ifTmCKsgi83OeTzrEccHF1niw/9RU6P
-FYH3jxUzGe1AxBfcAcgA8u4L6+Z/uHtnfU+y8od0H1kHwzAR90Rgc0/sCLlHo93+
-4yDPpS3JsLLrhrn6e0uHwEOTyoecyIXcyEL16E0CgYEAiH5uIY0v633u0MgEWDFE
-Lk+45OhAgiG7n09eWkY9Dv3TPATUvbXwtmigFEwywKyvT8kBx86uzWXVWUdgnjg8
-tQqJxZpJccAqdBCnWWElf/9VP3I/MFHrFJb7cP0e5RgcVDNUWf6lvjoHxd2DylJ2
-PvABhXMZ9dSphKdMOM76jOMCgYEAzPlIKSwj4qZVEe4cMGUIoKjjriN5D/LyLbQL
-KweKdjiBMnvuOWuYao/BDTeq72JhPDr1RyLF/3nXZt+HHAVTjC1Ji3doZqNufHeO
-SCHqkT2amqUqIwTAdAQPaHttZLAoIaS8k9lzft7dw6W3uPhLpEQAhMPTiBSVXagx
-kVS0fikCgYAhgIEyMauBbFeEA4kR8AuvpGfB8nMxJV7wljxGzLV1FJxmMsB+nkcE
-BgUUkMLRmqqbqfyTj4p8zNtsnjeNd6ka4Un9uYJoxLpbHhe7NxqGiMGaUMPszaKV
-/Q7vs7YissxWZxhhoTCMkd//YNikdHBUeMTYIC3CS6ahgX7+ueydhg==
+MIIEowIBAAKCAQEAu5iTosqln5jTOabUGdtqZaLVX4KAX/Xx08zMP2qdlZdP9OFd
+xhMmAnT7elfOxls4wUjvRgJpSEgUEMmzInmU2k5vCilttOuoiQyrHgQr/ZuJ0ph7
+9KA6qd0h7GUNsR4rOJW4u8LNrYXaqZo8OA7/eklIgiVI4XAV+JATEXDhoJNURRDS
+DiJ00ipQIsDRdTUp4Y/MDDcO5o8PdI6QBgLj7rokO/gGByQSarU/ZkYpEfKacIrI
+xBOpLJQgoNkQuOQIheC1zaewO18Juj9+Fyek/TyVIf696WtMR0biNs0C2lvGf2+j
+WJFBjGx0ewGP/ticzn5550HEgfBGAvWx2nbx5wIDAQABAoIBAB0jBpM7TFwsfWov
+6jOV68Gbd+6cs1m0NnpCDdsvsQgh904+jrUMFlQ9XS3UY45Vbsw+isNh7n5Gi69L
+1KHfJmp90itO4fY+v++BYzaHSVnbhZ2LB32oQVROv00bKPRAjk/8mTO4fv+bkanU
+BdRjJ/UTWsq0BczV/uObZQrJcJHi6+sAMYw4b/kxzTALd+UuvmOP7Z/NoWW6x8Mm
+ahHgqaMwA0O1f4DsdKYnSUVMF9DNGsxKCUYSYR6RH93Bq/Eo0q1U2egmLIMcTVW9
+7QSWsJoZuXlzkq7Hb7mxGdppa6kSzA/VM26qPNE9Cjg4tCMu1RJSfgkcnv27Y8vZ
+fZSq3zkCgYEA68VjIqG6sj43SZSvD+Z+Dfuzc+lO4YBSI0Yru8B4ZZq0vfTVQdM/
+uf0Bpk/nMbqec/kfcPMHP8zznLe8rcmfZXNQFIaajOb6rzWhCRSgbX98MeGnUe/y
+9sG+zFSRrAPDaVRJZwSYILs6o6Hz4o6DBCvr8iKFfm26SLB7hIjwx8UCgYEAy7EL
+dIMdsGDzfmxAYqad3oy/N1KVp96zfdnHEiIC0oiXz3YfI7YLFj54yXxx5rHR2/AK
+wOo7b90Rc8R0PgtKedKrz5p/E0Bz723ToTxHjsqgVRZqYaEKUOp8wR2t2DJOF9b9
+0C/qp6iUy0IOTBYyu3BCMV0aB5kRW62jXJIsQbsCgYB6uO7mOurUFsBug38wNpjM
+rIR3RCz0Afg/NipTe1bwBDwqWEOdFNmp9QEj0ZmU7//EfBsajtXqJsNzgswqZbWb
+eA9p77qItz4rby3YbS0oceByknOmmdCNEsI+15JPyFGyBNaEUgbhmrNmM0mgVu/p
+fvc8vS1hZro9VeelUCaMxQKBgFDgnXHH1fQAqu4ZwX7qNWj2bb5jtjSPgqmH3Tlf
+88rwnYasmjStxb0xVPh7xyYYmQFBUKPE3ZDPMGzNJnK0PQAeHEY0TByyzNXWv98X
+djpGTl86pUbakKQMVzi+thZP8x4YKXOOcxfbIimKsu6XKdGvAzlihEFcD75dNa4+
+BACdAoGBAJevnrC7M/KyDDGW3ci4sFcn7MxRGqLBulwGoCuM+zecbG7NBvDynoaH
+NRGpASiboRJyCEoIQivvkZf+K7L/oB4bL/ThF2ZpJUe471tq0444xnXdHRDLG0Dw
+OnBl27e3iAiUctqR51ufXKOUaNEf4gcsS9duELMPBxM70GE2Q/2r
 -----END RSA PRIVATE KEY-----

integration/fixtures/server-ipv6.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEEzCCAvugAwIBAgIUYTkp3oUkde9wFRkJA1LlvwFrZ3MwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
+MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDcRWZxskwCNXhprj8XCtkxj9GP4z9hVgUxgquSBync1hic
+or6qNgrUztv6nlALdQdf+TbPKyGEwCgAlKU/hnJK6lAG3+riyShnyM74/ulV1wYS
+F3Rkeh0nNCo95TPNq4GLB+sMfzwoSsT0srPX7KzCqpGy+G7sB0JBNwkTZLkCuMZf
+dkkmcZJ3zqIiOzJPlcQa4iBa0L1nV3Uuv49kLZLMCLMslg//IZxC09fnmjn+XLcV
+4+RpOKIn7AMN1kqPqmaB6gk2aCbYTZZ8aS9+cOJmTERbynyX4y4sRV18ED3dRNvs
+HCedgPOp53nqDneSOqOhhg+Mb95tnMQq1on0+TRDAgMBAAGjgZ0wgZowDgYDVR0P
+AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
+Af8EAjAAMB0GA1UdDgQWBBTFoXLQVq+Yg2AlRIirXj5ho0PMrjAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjAbBgNVHREEFDAShxAAAAAAAAAAAAAAAAAA
+AAABMA0GCSqGSIb3DQEBCwUAA4IBAQB4bl4f8TI7k+nlHe4MhJuHP1BKHB5O5SeG
+wrgI2+qV38UrKvTag2Z3OVKw12ANGN1vcOUrDS7cCtIZ8Aar7JpBgWrYvVlhAtc5
+3syj74Iapg1Prc0PFRmMQTZ4mahRHEqUTm3rdzkwMjNDekBs9yyBsKa08Qrm9+Cz
+Z84k/cQTBc3Bg6Xw3vUiL4EmeRQudBQAvh/vdxj6X+fwKmvLbPpgogXuQS/lHhFQ
+/rZ+s22RHLlqzAMuordjxS4Nw91dqYFwdYVvEmsK89ZnSWqwLvFCJ4uNnAe8siS7
+53YTpGbpLdNkQKAQJdMQSyvcDbQoQ7FI19a1EtSwpg5qSMOTpQ/C
+-----END CERTIFICATE-----

integration/fixtures/server-ipv6.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA3EVmcbJMAjV4aa4/FwrZMY/Rj+M/YVYFMYKrkgcp3NYYnKK+
+qjYK1M7b+p5QC3UHX/k2zyshhMAoAJSlP4ZySupQBt/q4skoZ8jO+P7pVdcGEhd0
+ZHodJzQqPeUzzauBiwfrDH88KErE9LKz1+yswqqRsvhu7AdCQTcJE2S5ArjGX3ZJ
+JnGSd86iIjsyT5XEGuIgWtC9Z1d1Lr+PZC2SzAizLJYP/yGcQtPX55o5/ly3FePk
+aTiiJ+wDDdZKj6pmgeoJNmgm2E2WfGkvfnDiZkxEW8p8l+MuLEVdfBA93UTb7Bwn
+nYDzqed56g53kjqjoYYPjG/ebZzEKtaJ9Pk0QwIDAQABAoIBABBdY5gM3BLJ8DFB
+zdQjbTF+ct5SztGnd2lPQPnvaE/M5DU27h1tOG7JE5TSEDZZsnuR412O4cWgFRi9
+8mz+yxz/vYRVPHku4r6bL61WGvXSrNPJRE92txXDjWPd1HRySoSOyQq7pTeFHo7j
+e/MN1WP9EigOxwboHycDNLxpHkmyV1DIlAgNkCZV56//liU/b+4vAVIJrgWfwfGH
+NkFd9nkm93oCFOroJ2f30E1wLPlC+ZhIn4ysau+zlWDLYeils0xHwS2GD7gjp/if
+i/ibVPgMVW/WPb67olm3nMUsan6CLmKWTiG+yklJT2djoam/iCZWE8/SAZj3qsxy
+6W9rafkCgYEA+D8tPM8h0oHlKriFDQZx37EH1dfGJRqxr+SgQiJ03d9pGYEsT+jC
+yr/l5ntzTwEEJjp/biIRwCwSWPYQtN4dNqn+11ICQzjhQbfWTfeT6vhSoBNxkeTT
+R8tUM0fmoUNrXhPbGZ9XdIxDFgD1pJL96KtyaQGjIRAhyG+khIT7oIUCgYEA4yaM
+Mw65KDonnKSVfMiOuG0QNYf70UcIiLSH8USnhbQhzT/c2LG7tNmru9UtQhZtmrpc
+vezuOYTkfcAIUjwqm12Ra8Px8WMzwHwKx3C2SrFCLFgjNFyoQ+VIGjtAL1lNKvEx
+MObSX7kVIf5+gaO9+KRBEdu55R16yQpW/UVAwCcCgYEA8XdqRkLoED2/Ln3LFW9W
+ZpJpH61BlCfR/FhzNcEUUhiUv3UxKA0tJE/ijP05nPhNE+5Es1i6UWXM9vFqMLP4
+UIqsUr73anGyUd1CvBX8sEqY/BHNn26nwKbboQHoKKZOknTX4qVmSPyB6K5IQaul
+BKN3pwIrreZmJfPKYAiGRY0CgYAYgEbtFvB321X8enA5ZnSmhfUSoRlTaIMOI9Lp
+/krHjDd9KR9MLFef2T7B4uufzkWCRAnO3qiPgbsXqUf8fsrluUD/S8JkFBw37elH
+u+udwOLvX45kjn4D3M5bLfrtYIeHUz7IFI2qj48s/INuvle2Yxk1sOqrQPPGjZv2
+c6rZTwKBgQCHSa+ToxicPJBZ5E7ezgue0LyRGWIMsr2OR16PBL2lPPiCWPH8Ez+l
+mTClHll4KVZyqc0VOZDbjMjZBnTiAq/1lb8ZvwsXLi0ue1obkkEYfXLWcxYD3Yne
+iBCGhjkqaUA4rESb22j7yqB8WGT83qV0kB9JwElzE9SxnyR9iw2FmA==
+-----END RSA PRIVATE KEY-----

integration/fixtures/server-revoked.crt

@@ -1,24 +1,24 @@
 -----BEGIN CERTIFICATE-----
-MIIEEjCCAvqgAwIBAgIUKhJ1faf58PSD3a0vfyrIDojJ/kswDQYJKoZIhvcNAQEL
+MIIEEjCCAvqgAwIBAgIUBwoN2+J27JtT6IaqV9sWhsHii2IwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTAxMjExNDQwMDBaFw0yOTAxMTgxNDQw
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
 BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
 ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
-A4IBDwAwggEKAoIBAQCczqeCNBDCkgqgJuhfIKlo1uw6NAvAN8tBSoxmYH6t51bM
-+XDJzuw8iQm8UuarHI68O2qjUSiVNvEoilYi/xSu78Yj5vRsNc6i4+FBs09WdSH5
-fieQ8kGVKRJwBDDNiMUQnG3DrEXH/cd7NLegyFdnO17yjTeAt8Fl482QKIFDuInT
-tGLBR+NKYcSM347Vz0tmJIuD+1Ri1DPhOrnrS0KaLKVvaDYpFVdNy0xS2zX+p4Ti
-QyznXjqXOXiFIDOU06huNerWJEzr8Z/ylRPz3pA0bGqcJpIXxzJcGJWN6Jfh6izf
-ONszTSgchoN9sn7iah8cHitEdYQs4LTKjyqu+9QXAgMBAAGjgZwwgZkwDgYDVR0P
+A4IBDwAwggEKAoIBAQDHS/zscOjq013InbTlwsBVwasv8e5+ukZNGDQx5RNaXYxI
+NVUM/5Bai4R3CS+DTbr+jBDylKi55gPQ/UIDKlU/NQH+x6UJB050G+aLDWAuRmxi
+w8dq7kRw2QJvuMxI+quiZhWk2HYjtvZRZLCUGl//QTL/VCT1smXwXRBU19S2uOfy
+g9KgZL/DCkJ9VBUh3+bFVKXBDnIphY4N/0+B/sW71cvRj8zvW3iD0R5T1J+QVEFz
+sFRT99/OhV2kUEwMaAYOFv/mMIEO6qc7vf6pB91qdUfEP8AbsOlmiSuOOLuR6X/2
+FHUjc8JrFfMuOVHnedRR5quxXbP8o83ilat0tXeVAgMBAAGjgZwwgZkwDgYDVR0P
 AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
-Af8EAjAAMB0GA1UdDgQWBBTdwPxrHxU94wYrwQBOjp6ma6EAajAfBgNVHSMEGDAW
-gBTdZTAnocY85lHhyR/A7lJkp3t2mjAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
-AAEwDQYJKoZIhvcNAQELBQADggEBAJB6rG/9kScmQi1TQMFJa0ZAsG+/9m7Rczye
-RApBF6pG5nf8FJiCt7sNYT8r8i+kby2H0CLII16dXSZxPG3giRN4TviM++/YXW/j
-rW1SueyhS+bxajOQfRVLxTnBk7TVDvacwJdFy/VI28i6hoV8E12g9jslAMiWREWd
-nhgk3zIyXFlVuiHIRYqKFWeo75/cEyTZ5XWs06r5Odawzo2L094CT4uxgu8mRCwN
-sJKa408ev6CUEW7YXZVtJ8IwtFfJCWAbe5Tsq/9K/m1puHLOiVRwYBl6rCUTLjGO
-+iKZYsV3wVf75iENevyv7rQ9OkomJokdWxhi5e+VxC4x+zwbvKU=
+Af8EAjAAMB0GA1UdDgQWBBS7gBJSFrjAHryiQpe38OMTzCKH1TAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
+AAEwDQYJKoZIhvcNAQELBQADggEBAE2tsLHtgF1T3d/anKf543q9uh61tr46HHf0
+RrGZF+RJuJY5XIAiCN514Z/I7i2wG/x1TDTKwZUebajbk4GvaI4nEnCXs05jwm/n
+wpdyRE1EUy5PkVFfXKCNQd096mpZu6EYXBGnQ2fQjg5zFvZSDnYaIf0vBF1WxE4W
+0a4a9na3N77OSamPEljM1RJ1Sk+Zg5yI+nwyKcWWk3OlD0j668Vp6/m5VZKyQEkx
+crfSj7kgRJWZRhMeh6li3xa9vDmzdF6ojGkgRN3Qljrs36JnmsTono2ETF8GIc+g
+eNByAQNppLJjMn+zsaG9J5pr0gDLubFA7oa8aAJgYgJMM/GecAg=
 -----END CERTIFICATE-----

integration/fixtures/server-revoked.key.insecure

@@ -1,27 +1,27 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEowIBAAKCAQEAnM6ngjQQwpIKoCboXyCpaNbsOjQLwDfLQUqMZmB+redWzPlw
-yc7sPIkJvFLmqxyOvDtqo1EolTbxKIpWIv8Uru/GI+b0bDXOouPhQbNPVnUh+X4n
-kPJBlSkScAQwzYjFEJxtw6xFx/3HezS3oMhXZzte8o03gLfBZePNkCiBQ7iJ07Ri
-wUfjSmHEjN+O1c9LZiSLg/tUYtQz4Tq560tCmiylb2g2KRVXTctMUts1/qeE4kMs
-5146lzl4hSAzlNOobjXq1iRM6/Gf8pUT896QNGxqnCaSF8cyXBiVjeiX4eos3zjb
-M00oHIaDfbJ+4mofHB4rRHWELOC0yo8qrvvUFwIDAQABAoIBAE2/MNKf2gd+vYH2
-iJRR720p0upwz7q4bzCqM7627VhwMVtj+gY1cG/7SjZzrCb++85cb10WalbBbQS6
-wkaLzseF3uoCIwJcE140OoWG0Dl/zh6T7C4zz0yTlq6vhTsDKyI3TT0Nd3PXYka2
-Nq2jZ6Kpj0QXvIM3mM5aCpwFWUInwGTAPG2tskhPiIcbnJnajiGdupia4Gsl1DWY
-H+L7FGxI1FHtOczb57Z58BQoG+vXFbrnuRUBWCR3TF0bGG+6ywUFPnTqE9TfVrbf
-1kORUZzIGbZBOYl0n/ylq5nc/DUXThKY1G6C31kTvZu6hTsgx/A5MeKOxu6eojZ9
-3DierQECgYEAwD1EwiKmaxOX9XksmVUa8IvbI/WKhXy1h9cbfJFpjPKUXBzgWAaQ
-Si00PzrzKFkItGsuoI8DoglZMWEWswv+ipN+Q8DqT6+RposqpIbO/Kvr6ynMjkNI
-+SW5Opfo1ih2YaTTSsKVYOwxI0HyIMnIbqUo/MDsrx0xM5x6ZcWN+lcCgYEA0NDn
-huvKY+NDYaQn6G3JjD+uvAqfsTaGSyPZpWpvNb8QBGmB9gmEHfvpO+fs+ANpk1xM
-T70piT15SQM8XlX5yt1w9MtCliXNUEj4KIZ60r+fD+LU61a0os3cUfeuLvck090T
-/8hg0VIfh5kT4vW+/Qzj3ToGJlbgxPjy4vTIXEECgYBbHS73rKCf45uEdzhSPGoW
-VNM6yegruJ35klSux+T1LwxuVCloz0OYRLCcv338Q6bCjIJ/Cwm3xSKHKvMtTaQJ
-XjrcNhVYS2iHPTKq5FI6o+mffiI701xE6kJOqS1sQ6VIRmz25B8U3x2zDlCKhdj/
-yDwTAAWuQfHN+n5wuh++5QKBgQCij79u3el10dsHDJC09aNON9WENpETmOr7XDF4
-AWAKa2/Av5KZQ9dCsHZmWKu2eJGcUuJlwVGwsdbV7Fr25d/M+o+RQxvXYB5yPhxT
-ED5WCy99Yo13mMyPYcv/U96aWXSmKxoCqrAUxOYe3iEJM184COzrsNxQxJm/Pj94
-SDAjAQKBgHCW9ndreIIEHm7pugis6/WaoZTkdU00nQw+pYourCZ4/zaI9QiUnFwH
-HPLkUdpt2fJKKu9h5DMGl1hzYHF4QRuPWJK8xGTP7YTJmZAzLtadiAhRaAq8/nWv
-fZJPEXDteRoceSo/Odujl3mDMhRTiGi5QKfdGV6WZPicLbvbRkmk
+MIIEpAIBAAKCAQEAx0v87HDo6tNdyJ205cLAVcGrL/HufrpGTRg0MeUTWl2MSDVV
+DP+QWouEdwkvg026/owQ8pSoueYD0P1CAypVPzUB/selCQdOdBvmiw1gLkZsYsPH
+au5EcNkCb7jMSPqromYVpNh2I7b2UWSwlBpf/0Ey/1Qk9bJl8F0QVNfUtrjn8oPS
+oGS/wwpCfVQVId/mxVSlwQ5yKYWODf9Pgf7Fu9XL0Y/M71t4g9EeU9SfkFRBc7BU
+U/ffzoVdpFBMDGgGDhb/5jCBDuqnO73+qQfdanVHxD/AG7DpZokrjji7kel/9hR1
+I3PCaxXzLjlR53nUUearsV2z/KPN4pWrdLV3lQIDAQABAoIBAQC2y+TVvY51bJ81
+lilJIIMnZTauCDqXdCVtKwkcxp8koG89/+Tdwj7WPeenAv7YcWBVf4U/6siDkgzo
+EJMOsjJ0ghstZFLkYBY+eyTPX9pbN27MfAQZ+Sc/VlxcuuRs/7aTgwzRIVXi1jtB
+Vph7j2GDj3rGJJit3w6PE90Z5MkPOhXwbPD+T2OCIhO0OQCv9YNrdHmQzFZJ8vn/
+FuKUjZuoKKnwgXvBVBKsUPvvSdPTWpavNYdA7WQtjpVYVjVHgEHZWtxUwQ43JHzb
+pABWqYp/XJNiGhZ+cEXsw5dBBWp/BPxbu1P2iagZTmNr/8EfGCq04fEkKhv22x0y
+FbQa+2e1AoGBAOCztIuf1Magca9mFD+3YZHgBv2TA2XSujwYBr/664dLjL/9NQIK
+00IBykiNykiWZ0ixcaJI1j+af7fWr5OuSzBVwdXMUZraKUEwrKI3hh764yX8aUYt
+JsqpAFhyro7smp3LaUyMCW2ZFVxayp60h8fQXcNepFwmK5o5BnsTsFHHAoGBAOMO
+ZooI0Yz/fzBKOEMM1Vdf3PpUqYnjCyJSXag8OeZn/OPgiYkwXWL0idfC49B0ArVZ
+/j2zMXJduIrwa3UIfd6tjPf8O24YOiO2SenkVkcsUwJgsB1nM1QlOamGw8BB+nbT
+O5V44r7vy3HldHHQgbPvjs0z5de3b3eBBTZC/2vDAoGAVNroSnYAV0YNyIwHB4zL
+9tegLDBRbylmFP2JxwQN39ji/Tm0w+Gsp9efOUj6Y/EQbf48iGlzJy/EHXugcGe4
+kzc/bOqswoqyW6DzAItxRc++6gBpDQxOAuhRbhVY4DZvqTlAuZyEjvPpgifzLn3E
+bOu+DOJ3tSjg/Guei+oCgs8CgYAUfwxKkZk4/SdiGJETnGj1xjWQc2wKgnBS3NSP
+h0BCyEhP2ckQlUkY0bJPw8wE2TQVYtZMg4yHImayRBmvKuER5ODA0ggbXByDdMUf
+U/ll215y7H95aAN+KQ4Xe47YIByX9WF/kLYHPmZDFc95JrVOpOVjKLgqzOhHBWKP
+D2U3OQKBgQDbmwsNr0mopOYiAp60KsKJmICUQO27RyL87UfdFysDrTZ+K1Pc6X1e
+HOFtma4zNftDym9Xjzz2eOXT6flHeJNu1qZwvurNV1g0JZdXnY83q4C130bAFJLt
+I0+I3vDpJt9wznYnC3jDI24gCbEJ2D//8dpeDNUPKk94rjsEjXxDFg==
 -----END RSA PRIVATE KEY-----

integration/fixtures/server-wildcard.crt

@@ -1,25 +1,25 @@
 -----BEGIN CERTIFICATE-----
-MIIELDCCAxSgAwIBAgIUHqSMAwCTIo4rAMwp1Dp8tBqn3eMwDQYJKoZIhvcNAQEL
+MIIELDCCAxSgAwIBAgIUEQuXXKtjueOgUpZjzr1ORDG/zHwwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTAxMjExNDQwMDBaFw0yOTAxMTgxNDQw
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
 BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
 ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
-A4IBDwAwggEKAoIBAQC8QxRZGCUocS7zTrqiOCIuM0HYoojgOZTK91GX8EooGHMs
-76bZrzTUncEvvdXxOOOAkSWzOz69937sP0G0QhPBw95UqZ4+vAcL+ByJ3PG7cKQ5
-HYEo5FyETLihD3mLXydXsSudv9sCGp8aexJ2Y6KOzFzY3v7oTc5evPeYKL7qnMri
-SJUUMr3W8cK1jpU/C+MddzYvC9mNe9QQ3vvRoNYLe6oKyHfV0XhR3pl8katZcf/8
-091KBzx35ksHiGJ2zmaR2IDs+HN+KMzKJiizPQC3VYBvqiSHJ/2ANDKk7pRwCvdc
-zyIafk679p2aInXEReOvfFbMqNHfpDAMIqG0As9TAgMBAAGjgbYwgbMwDgYDVR0P
+A4IBDwAwggEKAoIBAQDPAGzCcM5JSlitl/iHLYJ6eGO8MJ3S6R1qzAvmdB9+KsGD
+F99gWVTrJRzz/hJyo9Lt8GvNj9Ll3iT2QnXyyaSOX1+uT4cxBM2MFBBfERDh0WUY
+43uLQKY45H4zrS4tJTOuSGKM/LlK2ZMj++pQBqHsONrNG+nOhqe3qLqPDV3yBfmD
+PXfjASNvHINgxb9AwQWJydgjfGDiAwWHnKbnVScYBFgWfMG0Gm1wa8EfRfWD0NPd
+L61XwQwgb5VsYAs7XH7bxVbPm6E+/oQTOJXQHMzQYve9DFPy62KFSIkfvNwVRctL
+NE+k3HnyviDzbs387ys56ZjPG1/XpbFmeQuDRndJAgMBAAGjgbYwgbMwDgYDVR0P
 AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
-Af8EAjAAMB0GA1UdDgQWBBSJjfpommIxJQ3q67KG/4NQe0g2yjAfBgNVHSMEGDAW
-gBTdZTAnocY85lHhyR/A7lJkp3t2mjA0BgNVHREELTArggwqLmV0Y2QubG9jYWyC
+Af8EAjAAMB0GA1UdDgQWBBRyAdJmiDFhC3CYXPLW8kufz4zp6DAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjA0BgNVHREELTArggwqLmV0Y2QubG9jYWyC
 CmV0Y2QubG9jYWyCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEA
-d0BLO+3K4SfbHfn+rpNH2xCcXVyLZx2p/HhXo8ivzAQ/t+pkWBgxIkJ/qxHfQjZO
-O0/b5DC+No6UhKNNyunHxv/gwsmzz4rzPmCA0OD6om/yZoXtz3kf0ktmbKbO8RWc
-nQPZRC26D+cshebDAAm9XoCGErsdI1Xekx5Y66c1t1tUVZaijDB7FvnaRC5Rbzn3
-5WKRHkF+ahJDqkpTWUsFDuyRWF771riiA0kD5wwmaw5W5/knr1BiOePCUFaTZPXx
-r9LgDqIG7w1A1kbi5zLm43pN/S5qpif3DcSYYaCjQZC7yTVaRb1JBvcTY/g5whJx
-nyHrKafioCOq8CwHpbvZlg==
+sc0JnS1udx4HFStcPZXY+kyVwYPuRDv3GisO+TTrxzdupQQl4gPmuGa/cik4tKxQ
+o//XFgCBQCapO2cY+JerjMSOLHtt4YmdyYDSXeMjsRG0sP36njH3nHIYsFAoEyvg
+nJQX7iDWj/9bzMT4dU2ac3t9RgCtyABRoT4G/MNhWMlJt9XkwVTN2Pqf4TMV0GlV
+54GOScsWAIwoIDPOCoO8Q40jtFSSnehrlrW7x6B37gY/EbbYpZNrIDNckNfLJMvl
+dak9P/ovtjLk8GM11gE4s2ANWA3o5bIm17b1x7Fw122sB4Rxptbc/BDpv2GIY5Zm
+tqDculro2C7Ib5GyEBEl3Q==
 -----END CERTIFICATE-----

integration/fixtures/server-wildcard.key.insecure

@@ -1,27 +1,27 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEpQIBAAKCAQEAvEMUWRglKHEu8066ojgiLjNB2KKI4DmUyvdRl/BKKBhzLO+m
-2a801J3BL73V8TjjgJElszs+vfd+7D9BtEITwcPeVKmePrwHC/gcidzxu3CkOR2B
-KORchEy4oQ95i18nV7Ernb/bAhqfGnsSdmOijsxc2N7+6E3OXrz3mCi+6pzK4kiV
-FDK91vHCtY6VPwvjHXc2LwvZjXvUEN770aDWC3uqCsh31dF4Ud6ZfJGrWXH//NPd
-Sgc8d+ZLB4hids5mkdiA7PhzfijMyiYosz0At1WAb6okhyf9gDQypO6UcAr3XM8i
-Gn5Ou/admiJ1xEXjr3xWzKjR36QwDCKhtALPUwIDAQABAoIBACE9KrOMSss4KJT2
-uob3qDlF/YM1Nzt3jyjCv9o2p7Eql/NAINJgV9CORdSbDNH5PFmvxts9Q5egwf0O
-MhiUbg6Q+YkzJWhoJEpdtRQ2OUvh2GzpI6vHnfoR3as6IziTRUv7UPsaK53Ue3De
-d+UFNGdH98CmppT3X89kR8bKsuKTHLd0bL3gPt1kDqyMGEEZ3XUB0l4PrakRv466
-WfYuT9suHjhdsNQn0bHzr93OyWT0KFffrwA6l4cKsGgf3eW6CKZCcqzTRHfe4Iip
-Oa2l0O4uM1Qp3Eo4KPBsxk2i6dlLrAEhDUUlnvg5PCIResWV9MbBERKA/+BEi3CB
-azUiyEECgYEA0Wk3AWWzPZnOVdGMltiArllkSODuwekuZN5VNQKYs5Qfi3fHcm9n
-rXN6/wSOai9Mt6oL8wGHODyfChmllgtU67jV7RKQUy0WqRN81KbHrQynqmG8DAgS
-aR/np8pIYoyeL1sELH9CvRSmLfGWxa4j6J2wuUzlCPFeYhmniSecxjMCgYEA5iVY
-5rUXY8uILYkVBme8YEPbiqfTmgaXcsVhB9H331o5BX8pFXc2n0+igbkQ1ZomDN/u
-5qJ3ZLk/ya77BixVV7U1+UPJM3knBBBbzB6zLGsXSpMuLf/cGOM8nnJ2EFfl0QnB
-nTcqwH59iigUHBLXnO/nijdZtnN5P/l/U/ZDcmECgYEAojt01axIVCzX4TxaQnKw
-HmI6gwtfbPKNcq+cK1k7m8PhPFcrOMh99a5FV1PcUP8b3B3s6/H/I94zB8wesENP
-It8rPGLpVMbVi18Bkm0yvCnVqvXUjS2jtbV17lOUCGUQF3fXn/FnbryUkXtcZwEl
-6Ixh4Oxlc6wqhq2BUYxStGkCgYEA0cKwaHYdP2O+VmJeu1vJIaQ3cQTNo9DmMEEd
-0tbYqMW+uvfPJjVln4Yhg0J9yXGZxJpFUAg78Z22Ocg2GsZFco9DBlF2DGgb22Rd
-holknNNugxXqPRq6LCTQl0mTugmi+Qd/ZB9n49Jl8Ynd6khyJCO0URFpvxU4Kcro
-9km554ECgYEAyqC8orPs7VkJJ4CCaJXGNQycFxlklv9vodV2b5dhOsmX3gzs2Ssn
-xhMphgGR35YEsMWxpKofiMbqpI/VZEpjH8Zcb8IklVCGKi61SQvBgJo7/545Gr4+
-ZdJ9TBLTq21LhleqLMFsWMGR3v+5Yuo1NoGf6jxJ29J1PjK2gooqIw0=
+MIIEpAIBAAKCAQEAzwBswnDOSUpYrZf4hy2CenhjvDCd0ukdaswL5nQffirBgxff
+YFlU6yUc8/4ScqPS7fBrzY/S5d4k9kJ18smkjl9frk+HMQTNjBQQXxEQ4dFlGON7
+i0CmOOR+M60uLSUzrkhijPy5StmTI/vqUAah7DjazRvpzoant6i6jw1d8gX5gz13
+4wEjbxyDYMW/QMEFicnYI3xg4gMFh5ym51UnGARYFnzBtBptcGvBH0X1g9DT3S+t
+V8EMIG+VbGALO1x+28VWz5uhPv6EEziV0BzM0GL3vQxT8utihUiJH7zcFUXLSzRP
+pNx58r4g827N/O8rOemYzxtf16WxZnkLg0Z3SQIDAQABAoIBAQCd5u0PxY0WSygq
+A2sJcqW9Vmh9/XfmkvxloxDQ0nPTgjnrDiLPFFW6qazUUlMwL9eOuX8CZ1uxDSuU
+zk26ziZAlHAgP3oY4lkJKaTzX8lI+Lntqllrd/1UGLhMIya+OUqa/4xtj7qoZh/f
+qyKpuOV7lEMTgt9vMzhs2MC2rrOjEZxcpuwpnZLKvpuwBMcxD1ccRdCA1zHvKdQ9
+ukPTRVjz9WUEOgANRkndHTZKWMz2p4QC9Id35HlksZi0/M6oboz2Eg1mtZEpbgUX
+loMv1CPtWP1uj9PFWiOmnBC2/v/2MVGg2fJ1Lf4c72ZVFEIU4l3YNiV4IqFb38F+
+GJVcmiGhAoGBAOvrTjYYl5vodK57gRRT0UsaU6x64/IK2i0vbBGTATywuijJ313X
+vwZBU9I2rLZqr7FZ27g5ANorw8dUKn92otr/TVS/c/VZOSw/+gTM9Rl4ZGji+qKt
+4zY/dA38jlDJNWmFwK/9KNOfXNS+WLsA2QJlONgUfkFPb3yXJUGLsU8nAoGBAOCf
+AUcyDHjGwtYsLc/4aiKtQUIdeX0v6jCWtl9EI9cZ/o414iapE05sOGb724itSFN/
+EI4biQGw8CaMcaqMaRJ8+xVQQJ7qkXItzZEFVGqz0PKwiwYAwFp6raXuio9y+cTw
+savJIM8IDijph9ezRCalef4Qj6I0zFI8H7PmiwwPAoGABWvY1kFmanzDAadw5eiv
+LIykU5hXWJ6LOPKYBydbpethu8I30c49Y4VoybHb8i0tcGPiOq+Ep37N9uymNVui
+jmnDeykTHxY3zB6EPkv/beBoXkio/cgFKp/2qMOe+ZhGE/Cw5tpob8R/u5vMKi/w
+zK9KyRxfclzC8RgAESuGnY0CgYEAlHC/+Xrbvx0rOTps9Blomo4AqF6uIMr/ayjO
+UNrJDKfDD9wQHhhyB8uA4p3ikMpjF7rLB/6uZg22RuNdYqXz8iHiFE26xsqhX+Fh
+DkuFZBZ9KUT+OvNYKvMTuqqPqwkCguHFqI78PZVHNkZOXX+8tAV7PylWoo1d0aKm
+GM9saIUCgYBXO9TtUTiaoxIVTe+r+Abt5iwasAwxai/RvymdykwYFwq2NOEl02oi
+fU3gbqDV3oirHAsAKJipnrASc70hTn4SM9hUKTQrD3fNIABch811ZDd8vaHEzLZG
+pp9yKam09sPvQo6O4E7TJPccrddV286jZq+qO9YNIsRlYJiCnjzihw==
 -----END RSA PRIVATE KEY-----

integration/fixtures/server.crt

@@ -1,24 +1,24 @@
 -----BEGIN CERTIFICATE-----
-MIIEEjCCAvqgAwIBAgIUDJK/Om7oMHhIrvfvj/NMLa2PbkIwDQYJKoZIhvcNAQEL
+MIIEEjCCAvqgAwIBAgIUdUecO/Un2wfrJXsZRGkX2zFkK7wwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTAxMjExNDQwMDBaFw0yOTAxMTgxNDQw
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
 MDBaMHgxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
 BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
 ZWN1cml0eTEUMBIGA1UEAxMLZXhhbXBsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUA
-A4IBDwAwggEKAoIBAQC20alHsyfXrOesg+du8zV5OyzCkwSfXrtDG+eXcPd/y6g8
-uw0MQq4A3aoV2mrpLA5BOJbfaXQEoTutM2xwsNubJl5jWUr7FtFDlnvv0uv1jRkU
-OG/hEwpam2otW6yFGT7YnVtfrCemvzk66siFooofmSWM2aw1r/U5irqtsR3S6dMY
-76YI0ZqdPYU82l7aRp+mEKPMjt7S4renIuK5Y9GUwJAWoCanq6z7rU3qZAWUkZJ0
-cHAimHhXG2bBSWbCS3AedmosWmBrowACeVh7TCwvxd9tz9WMFUbMHV9mSgViJIlS
-p4hdL24+5gX3QQJgswNlSdJVKBeYevHWRxp2pkElAgMBAAGjgZwwgZkwDgYDVR0P
+A4IBDwAwggEKAoIBAQCgCUX0Mjoo9cckyzK+OdFaUXWyB62pk+tRi74OCPeym56n
+IF+eQd0llbBJ4AfK6udhc76B+eGFu/tgaroZd3PXJqBCV5n+EcUSYxChWZwbbmVv
+mv6fr9kdhJNaF0LJ5IjrN7bVWHwIkEQsaxs+50M3yPxsjC+OcSGR+uSFz9gf2e6+
+bzYvumxkv0HuIHnPJ1fPJn5Dyu6odjPjoqKjY3opVIuMUyrtAlr/XsODXxWLhoQ8
+KkEGZA+ErDiVp29rNH5AfKytqF+iJPzKcHk5ftINYRx482NgJOWraY2j+KrKkv+T
+bv9ZVQ3GEEHvdtxljUPXeFwY/qDoxQFDWOvbS/8JAgMBAAGjgZwwgZkwDgYDVR0P
 AQH/BAQDAgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMB
-Af8EAjAAMB0GA1UdDgQWBBSFhLSU/LEPJNAzk1lir5Gekdm7sDAfBgNVHSMEGDAW
-gBTdZTAnocY85lHhyR/A7lJkp3t2mjAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
-AAEwDQYJKoZIhvcNAQELBQADggEBACgZtM7ygZSzujOQdPy9lpC/9TJkGqNN1MBg
-+SJGalygPY5cSaVRS6ppWNnLgvy+mIGTBXfxH+2eVO5frSwzTCB+seXJPnzwxjhO
-yj89MKtPmy1YQAPlBvi4IOKJJS4cvYKhp3Dj1ca2JDOJ4Zb/FYRn+pjy5L6Ash1Q
-0pqlt5Xty7EU9EeUgf+2wHtrsCb4O9eR0qPr500ZS/7NA66vfu3vv1vnetIuAhFP
-zeJzL/9DBh73lwxoYroq/M9AslLLJUhBBRosxJGU8b3e2ylM6RRkAInI/rNYPlI+
-nRA658kmyYMM0W9hUB6mTYy19z9nZv/KzF+cYm7VyEmiGSHCD9I=
+Af8EAjAAMB0GA1UdDgQWBBQtNVqbrSLEx7TA7E4KhsH4h8NRhTAfBgNVHSMEGDAW
+gBRlB76vjaZyFLrEUGm6DQfyjmN6PjAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8A
+AAEwDQYJKoZIhvcNAQELBQADggEBAFdeFLblwKDjQ/aTUe0XhsjI7IdKc+tQujrV
+uU15FtRzA977ntJCYhaYyNpTBjOo65C/UOyZK/ZD6vI6vH2ENRvCgZUIInmfKw2X
+w7TbTHKTDZqbnKVKqORx2+S8NBaE7KSPae+Q0zQbHLyv5btBcRGv8MECGsi+pjP8
+EiJM4VMQ9obziCtWl0j/BOY+YSgjIAR9huyJEA6RibCoNhjCaSSSzNqIEzpCh3zh
+6zR0NzQbC3z+B6IGWzvnPBbrc2LH+eqywSlswVy1Thp2btU8b21WcjIqlcuL4Q1u
+NmHmq8K2zLLvYQ3voLYzxbeMlsuZjCI2/UJSNKOSGlmlNswHy18=
 -----END CERTIFICATE-----

integration/fixtures/server.key.insecure

@@ -1,27 +1,27 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEogIBAAKCAQEAttGpR7Mn16znrIPnbvM1eTsswpMEn167Qxvnl3D3f8uoPLsN
-DEKuAN2qFdpq6SwOQTiW32l0BKE7rTNscLDbmyZeY1lK+xbRQ5Z779Lr9Y0ZFDhv
-4RMKWptqLVushRk+2J1bX6wnpr85OurIhaKKH5kljNmsNa/1OYq6rbEd0unTGO+m
-CNGanT2FPNpe2kafphCjzI7e0uK3pyLiuWPRlMCQFqAmp6us+61N6mQFlJGSdHBw
-Iph4VxtmwUlmwktwHnZqLFpga6MAAnlYe0wsL8Xfbc/VjBVGzB1fZkoFYiSJUqeI
-XS9uPuYF90ECYLMDZUnSVSgXmHrx1kcadqZBJQIDAQABAoIBAEehUmBDc+LvXj6b
-1/wC0vrTErCSBgejiF+8+Tq/ClpKI3rYFz6siZzRPSke0TDXECbB02tp2AlQWx97
-vItS4Fij6eunOteykv+WYK0QyV8Gj8L6ceuQGEh0CGAJQUlNIPCihwCvZUHSPsw0
-3ahujQvgZ3QOfYjzjnOjRY10jijqxxsOfGJUjgFsD1ltMMLCrHBLnBk/KXgWfEv0
-pJ79RDhQhZHcB8T5FmAc8cPnXj+uZG8rZJ+VHQq6CM8/clwTb0SBk0gdWuO0h8/L
-0k4qQzZTLnB3ASS0TMbyLtdnIuCnIVgXYT5cc0THPNI+/2gdklUl5kjUviNG9gXT
-PFujFcECgYEAz1tFvIs7AFS9Y1usxmzd5rLReninnyIiGORmFMDT5vg4eQW3yzT/
-4NLVn1L61WKxrckpb4C9J4rDVVrDbV4RkO3CM7Fkeko/bvPwJj5h/aM6pvguMD1v
-UqMZGr5QcaYakabFt1ELUTURGlkkvg6fsvoac8F96Axeu8QhEIPLZRkCgYEA4bTL
-ebP5I5rjiq921JWm8N6/qnxT2cnWIXpeXdYWnSvtYfQ9LrbVt17xyJ+niWmZMAD/
-dmdWbyrTTBGjsAUT+U+uX2sup+3pXSG4nFrRZ09n+vNQZa8ZliyTg4x1fyjJmoXj
-/yEuFjT5pwghE2DroWy30cIxWB+cG8h6/4gbEe0CgYAX0xreIPk0fogMJHpjihqs
-6RrcgYRw6lEUnxmDhOxT+20xqpCFjp9fy5mz5qrfXamgmB5kq1wQcQckhXsy074c
-8tR/cABldKZ5LxoGquBDbj/M83MLI+PokMjPo9JGXJls5saM24j7WaePaXKpS7DZ
-tZbYegDnapRv6ocKCLqT8QKBgEINMbrFA3T6//n3DXR68ybsWPTge31xxrt3XyRg
-4a9PSqHu4vTHxtVp8KIArvvUrmLQ9/HDnhTcWIebZea+JbFBM0tzR20Xf8Kkeq0v
-Grb8Evbqu612R6ueEfFeaogy/IS/CBECucT/7cuG4n8UTwCnm0fEZ8JdRccPrYcY
-YqulAoGAWyNwkmFOooU48FPFkt7OKcXD/fLzzcXTeW2fcje9+bX3Kmkg9BE2B1Wg
-MDCMpBOKkUB/COtqS17iC7zdViyDRO0rxybBLFNAVEq23K7/0zVsxuWCEIPnjeE4
-nbh4EH5L/heT1/HrkiJtXvNd3hpYDXiVD0b06iYBVO19p3uDuO0=
+MIIEowIBAAKCAQEAoAlF9DI6KPXHJMsyvjnRWlF1sgetqZPrUYu+Dgj3spuepyBf
+nkHdJZWwSeAHyurnYXO+gfnhhbv7YGq6GXdz1yagQleZ/hHFEmMQoVmcG25lb5r+
+n6/ZHYSTWhdCyeSI6ze21Vh8CJBELGsbPudDN8j8bIwvjnEhkfrkhc/YH9nuvm82
+L7psZL9B7iB5zydXzyZ+Q8ruqHYz46Kio2N6KVSLjFMq7QJa/17Dg18Vi4aEPCpB
+BmQPhKw4ladvazR+QHysrahfoiT8ynB5OX7SDWEcePNjYCTlq2mNo/iqypL/k27/
+WVUNxhBB73bcZY1D13hcGP6g6MUBQ1jr20v/CQIDAQABAoIBAD5l/zmMj/LCiehF
+tj5HauJtWpeUuNiizSDZfLwaMQIZ/U0qqT2abrCl4bucN02eM6Nirsgc9xrexc+9
+LVyan7cm31uernNK2G0n5ScUOnLTo4dVhqwas2v38kAxS6BOlDgqXAZpXssz/PDY
+viHTp/jLS+jC7BP89lrl2U53UMxx3em+FIdWmZ54wNcLrxD585Ibcu2ajRS2PQT9
+szh8W0fEkyKCk1MOhCMFvXOuO34BqO7kMev1QTQFQ5QeriQU/XB1MWFwKldX9C3b
+0l5c4zJ1msTXpm/IyD21JtYfCJiH3Ny3Ojl864Ud5pdG0xQ2d4qHE6fvSRvzz9aG
+wAntmFUCgYEAxW4BZctTwrNMZwrXV4kNKBhkh6zl4BMxMexB0ze0N3aJmPEAcJUb
+7uoT6YiN3XA7DBRYOf7MmcpcLyqJi5Yt9T24xGIbU50jgxMw1KMIHf0lbGKQxDJN
+mdvLKM56O137qvnhaRYOX73pq2+DWD4UT1C/SWF+dkMTuG5aRz94h78CgYEAz4Nh
+8IRZMET+7dg66qpGqZ37Y2igANrxvMwKmY9VlAbHRZu3fpt8WwQ2Vs4CFk/rfmbI
+5H6mAUVu8pKYvi910wWvLlPPh/34Sxoj4TJ6QRlP0pD3Um8O/PYa/0joNlf/rZjw
+6O/7UkXjzubOUHqcEnYWEm+oym2e0Av/8yLWazcCgYAvZgej3rrPRaiUHIAync6w
+z5pjEFloAHORHr127iqwHh9Ovp1yafn40+3P5V7ZyPYEImZEFi4cxf53vGilQHrs
+I9NWIo+Y9WLvNw5EHpf2Sy5O5SMIV2NWCvStaVTjJ98h3zgEuKzew4N0CyOnbdAG
+csZZ4bQwxE3Zu3SlIlHXlQKBgQCabEj79WFxvEaBtMHTU7eWDcy/o2I+gLAYMTdK
+IxIqQAkW0dRxUT/vc2kEm/WNqRe0TsT81QqwM31m4pTsIuFpkfdVYGU17FdTfDZr
+JWc4/p8aMWr7W04qDPL2Oskjd8T66K+OiNfb18q5c6Tg2v0998ZhHdrcGUtvwx5L
+TweFbQKBgFOC+y+2mVhlKyUxibiGmN8LAmnFYmaFyHgvQDCOOIpo0VAILf8LyEQf
+EYQteM3us92jBUOuZ8NMUaI2sBCzaLPpZTkL02iIr13Y0IYtu0LgIfI157kXjoow
+ByRl9TClp9RAkmbJVCxDsgR9nJVW1L1cCw8M1p5+n9SQMJFUQghW
 -----END RSA PRIVATE KEY-----

integration/fixtures/server2.crt

@@ -1,24 +1,24 @@
 -----BEGIN CERTIFICATE-----
-MIIEEzCCAvugAwIBAgIUA4E1YyD1y228iRlwJgN2TpMYiXMwDQYJKoZIhvcNAQEL
+MIIEEzCCAvugAwIBAgIUCHfFkPZDhLT2oK0fhu2TP4xZtcwwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTAxMjExNDQwMDBaFw0yOTAxMTgxNDQw
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
 MDBaMHkxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
 BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
 ZWN1cml0eTEVMBMGA1UEAxMMZXhhbXBsZTIuY29tMIIBIjANBgkqhkiG9w0BAQEF
-AAOCAQ8AMIIBCgKCAQEA3Jo8Uwuzyj5BU4Rx6eBA0bkIccRgruUWpZDQ1e0fO9Au
-WgYcRSE5O/EiJW6MpHKhlT3jhBw9bmzDrcAF+Hre/1hCfRPP3udDerYAEbpHEQ4a
-y1c/FlsRqaOPWS3b+Yp+RQCvyDQGiJ7TPifm0vU4opeeM+MUFmnFOdDxS0FxJOWD
-JCdCckr4Srghn56srwQPJISzS0q73YT5hv1lgg4yr3PHtEzBYvzw2yRxY8nEE6ZM
-8hGnOwjDn9FNRMPr13wd05w94hq8akDHqvRp4yHKXfsw11fW7jUj+1D2GsHL4VVT
-mafx+RJsHy33/Lou5eJ5HIOeYsjX3PDs2KS7MlZX8wIDAQABo4GcMIGZMA4GA1Ud
+AAOCAQ8AMIIBCgKCAQEAv6FwE/OeAwAmZutb+DTP+y7FKpONm0+aHtESlrmTRmDo
+iLGuoEXjYcFOCHr6gQJTYKEAivBK3TIN0dRirNBqLQxssDlsbU4ZXG++OciH/OoR
+8E+VmMsbqof/E0nhVFYDumnuoS+waX8elzuDjDX4u7F+d1/gIb8aYU1VDjtZxF2b
+vqaPyroOn5HvBs4MW+BpAB4guHfDXpK/oAnJDsq9JTUZqoG1xOZfHNhA/iVBSAJv
+hO6aBxDCjzwO1gT5kTbNELrBCJ0V2NXlHFcBOPhxyl4+DIPwa0Q8oExUEOzxnHFu
+U3GPoklbLp0RNNiqHHKfD4yaLI1rf8dH5AZG/QjK4wIDAQABo4GcMIGZMA4GA1Ud
 DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0T
-AQH/BAIwADAdBgNVHQ4EFgQUj+ruFZbJNhjHRC90urHmkegftsgwHwYDVR0jBBgw
-FoAU3WUwJ6HGPOZR4ckfwO5SZKd7dpowGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/
-AAABMA0GCSqGSIb3DQEBCwUAA4IBAQCcn0Y/zym6tla+ku/sMiAMrvB1NZNVd1eE
-HxPns99H7qdlGQ9bnxf1D1WxKu9HztmG3DhfdYUxYA3Z772+rrx3AhVQYquZAvvK
-kdZ6ixHynY90Q5PX48BlgZvzd884RZFDjwSCoGkbdYqTtD9FfuGS/fdYcHZJvVAJ
-1oRXH0D0e3I1piUAzfFkLTXqgmBHh1zgNHqq3CVcjA/4CVGn2foI5r7yH9TIO+Dt
-hegJjObxJQJjobDJ5ijXbkPLt+LC7OFEDGbn8IKBIMmLYXULGIJR8wLKpiaBLV6G
-zoME6J9TZe2g2VQYcPxwshesIx8MAcqNVjOBI6pBey5t5Lqhbj4Z
+AQH/BAIwADAdBgNVHQ4EFgQUS6HUgF/GO55DtrxLN2dcyGACgnQwHwYDVR0jBBgw
+FoAUZQe+r42mchS6xFBpug0H8o5jej4wGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/
+AAABMA0GCSqGSIb3DQEBCwUAA4IBAQC2pcpQZVkvZO4TC1zluGhG6Yhz+fzZuNlp
+jHa0U+WIdTPerqZZ98iU1hCm3F6voNjAQ0eHRV3z5q/rDph8AR7cVl9cT2rLO/zA
+F6M4QDMetddj7EIUq2/B1CWzHkOvwcJFgc0OfBWzJYAShFv/B7Ir1WpdtixOvyOH
+kWsWoy1WmatQvBQ2jDrvdGRWhqsPmg2uGbJrUjABeYtc5whQr0zscy+jEIrDpqPT
+VVuUu19/ALvdv2kOC+ayhH+vTAvEA38P6wlavDlgsv/M902ORWahdLQ/H0XX+gVP
+QDe3MyrBR6QkjAfKJvnMv/5x8mj+AFWfnALnRb9j3/q1UOwITWN2
 -----END CERTIFICATE-----

integration/fixtures/server2.key.insecure

@@ -1,27 +1,27 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEogIBAAKCAQEA3Jo8Uwuzyj5BU4Rx6eBA0bkIccRgruUWpZDQ1e0fO9AuWgYc
-RSE5O/EiJW6MpHKhlT3jhBw9bmzDrcAF+Hre/1hCfRPP3udDerYAEbpHEQ4ay1c/
-FlsRqaOPWS3b+Yp+RQCvyDQGiJ7TPifm0vU4opeeM+MUFmnFOdDxS0FxJOWDJCdC
-ckr4Srghn56srwQPJISzS0q73YT5hv1lgg4yr3PHtEzBYvzw2yRxY8nEE6ZM8hGn
-OwjDn9FNRMPr13wd05w94hq8akDHqvRp4yHKXfsw11fW7jUj+1D2GsHL4VVTmafx
-+RJsHy33/Lou5eJ5HIOeYsjX3PDs2KS7MlZX8wIDAQABAoIBAAurkSNnjBRX5MbR
-S+Fufp9ZpYu3MtItxlvt5E5c4/kbE0ip6Bb0If38zKykQ4Zzlf66Lm4PvGQ6FsP7
-U5WIkoF5ntLVGHsXSueT9z741sedMmetbuX03WNXBr9WALwbyoBGAAacUfgqRsyT
-+c8RL/TSBomCKs7gu5FKb2lmxeCItpeyxGPp35AnXR1B5Q9GK7IoFnYlpJO5Vqbs
-t1cpOikotKTFrylq6hxGbRBtzBnrOLh51UneXR0r83oGzSrpHm5+BLMG5orYnK10
-UKBge1HijLjG9EDpCLdR+T/IJyz3oQyFCPRVFrIq5/76C8LsMSSjKdM6apCMc9d7
-NMnHPYkCgYEA6AWosramFnuAjihEkaBgJdpFRHfco+FlpglcS5cvaORmkW7dLl6R
-SZaO/mRJuhtC3j+LBXdGaP6rMdW7it9wRXs160e5mfSrEb7H66bgD0FfIG+iaEPV
-8OXESgrSUI69neddRgL2QXZeY++hSzaMNGef2lPpFd+irgkud+bltSUCgYEA82Z1
-/VVubW/lcnz2jBehssXL5yqNa/ySVvEWHdcKG6b0RNsoQCmOUZnBIwqDLMT4+62v
-e3bYNi/y44oJi741ifT6prPPQqv+J3tsEAQ0dlcscs71q4TXJEpLIoSlM+lQAsYA
-IhOkeLA5YKr2RzjnD9m2d+awErK6GGtCQqGbqTcCgYBHzgiJVlFMP+hnFjsyPknD
-LSumptmXthe5LMhtdFptwdGkTIUS5p1cAsY6IFtYxzsIgO3LQUB/GeFtSNIDhma+
-egUTzVy5MqkGkt5YJYrN7dM8vI+saOH67YCz0WmJGMPB7GpHUn9XfwRzNSPbnQQC
-69biwHkwFcfIyHqjDMgmKQKBgFWsTwOouoHSzRSLX8zX1Ja0gJ0RoU3NTUVE/t/p
-/SjWj0xdR8Gt3uZiFFVdMebkPi94ZzgyENCh+ACXadzCSt4QWNmsmNuC5qbHDrZk
-hILTFFYk+twwmfmwHNo9jljDWuJfB1T3TQEeJlQcWSugn9Q4cb2qeXdbaZ2Gw3/o
-mn3pAoGAbrBBySSjr8AnCmW5ipJ+ys1f8Q5Sg3Skvvx72KNH640AFY9c94glumjZ
-7Ryz1BcVIVszaJt9q21xQZeCpQs6VBHQB04AKlJLq/wOccg2Qb4qD8++zNyK5Ndm
-Bht4nGrX+sGDD4EwLKUCTNgxBgQju6ZQrT3b05xlTsRG0fA7SQQ=
+MIIEpAIBAAKCAQEAv6FwE/OeAwAmZutb+DTP+y7FKpONm0+aHtESlrmTRmDoiLGu
+oEXjYcFOCHr6gQJTYKEAivBK3TIN0dRirNBqLQxssDlsbU4ZXG++OciH/OoR8E+V
+mMsbqof/E0nhVFYDumnuoS+waX8elzuDjDX4u7F+d1/gIb8aYU1VDjtZxF2bvqaP
+yroOn5HvBs4MW+BpAB4guHfDXpK/oAnJDsq9JTUZqoG1xOZfHNhA/iVBSAJvhO6a
+BxDCjzwO1gT5kTbNELrBCJ0V2NXlHFcBOPhxyl4+DIPwa0Q8oExUEOzxnHFuU3GP
+oklbLp0RNNiqHHKfD4yaLI1rf8dH5AZG/QjK4wIDAQABAoIBAE5QTHxq4BV71zXS
+U7ig5KpTV9JpkMJ7CpIzgTRFzNFDQ2SxsJrhVOabWCeREpTsfWSNB6rAPugcz5cE
+A/t6BRo57KUsIoqdEzI6nHQC5shOZFxgOdPClaDgiTa5x7Nun4FsT1BiK+dBQyAs
++zqux+L0y6k/blp8Peyr7OmvCaV8osB4/JLLH/WHt2wWgqFWisyIT7/D/gQlQn81
+Hvv84BAL9y8iyCmCzWhQL0YisLPyaFkGkb7DK4wznWxfQn3jRAkZDQMH675o/OHj
+8nL0NSdCA/MGLEtPAXegM7kMPCf68JwZV3gPZDyEK0JES1oT1z+op7JHuatlhgdL
+WTA10fkCgYEA1QXRRpOeZvHzGVMzrXrrgS5GeaR+XgjUalBGgu0w8nSET195oXu6
+Y8dVco4FlEZ0Wq7evA4M9XVJyKQkkEGR7Nkv922p8RhG+U73ajODANAQURIwqOPJ
+01IrfMIK2mkXDZwzkAxPaOnny2OMZtUznmZnNdJ/vLd7U0sScNPVQp8CgYEA5krD
+ImQ8U9/S4VOK78i3FMWMoutffXpW71lEc9tsz1YWUPf170raujjF5mqtBBXup9ko
+37CmVk6mOO2TdXLg1feMaVBsoblL6iPoBZot/fLdzgmICccpimst2yrUZEHwJpdk
+9k95xEZUQhN73eY/Ih6b5HZZ/ygxfAVvhDFzNT0CgYEAru5aDvUGbU9e7HsQwvNg
+FfMkWJwmUZ46oRtO7BFP0qqwRGYJAf0S8QEuQCY0mrDIt/dGXXPEXIV2k9eHVxch
+eDhaVXuuxJfFINIiBwpKGA7Ed27Smr6EbI7bu1W1h+oozjppdW9GfscmXDVhhMir
+3PYG54H298hM8/eAKzsps80CgYEAzk5Vp76ySNVv3spv4kYmtaYQSnef8RIjRYLs
+DvqY7NmLXnf0y618a22m5LfWTZ20Uov50QM40ILe6Ir1GjeS8jw1frc8yljsiFIo
+brRj1We4ivcA9vmD3mwMBZbF9RcZJAlmuj4SsOHsY9F+mxjEoDVZpP7duvbv9dIM
+yBlgw2UCgYB27Zwd3ta3x9mx2wRHaFMlcnwh+ERRGO6yAKndsr3L80Gkb2HxHlR3
+cA3dIg/YAGocjROOIO8MO5rclZm30qM5rlv0SH6gQ4nW16PZgHSuakgeX5fVHb4v
+nT8TKt0PY9r5e5cdHzpwlrDD1Q8GiAHvn0oH6oioGxlIV190YAHvGQ==
 -----END RSA PRIVATE KEY-----

integration/fixtures/server3.crt

@@ -1,24 +1,24 @@
 -----BEGIN CERTIFICATE-----
-MIID/DCCAuSgAwIBAgIUXO0yz2xs67pa/mT7ro2mx5d9+8gwDQYJKoZIhvcNAQEL
+MIID/DCCAuSgAwIBAgIUMgfWSv3FrD0dRcHmPNxLr7p9BXcwDQYJKoZIhvcNAQEL
 BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
 Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
-Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTAxMjExNDQwMDBaFw0yOTAxMTgxNDQw
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDgyMTE5MDBaFw0yOTEwMDUyMTE5
 MDBaMGIxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
 BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
-ZWN1cml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALf9KzBo3JFZ
-1Zi8lH4t6MSCx/de1MyHSR4S0sD+rl/GcdkdPLOLGKikHZLMU5Z9u1y5HK23QWBy
-KcdLFDkQckoUBDxAR+sxdVX5RgVTkMR9zdv/4HVertakKNutrno0lUdqhCtV05YR
-HeU8/tPMciyE4cBbeSLlvxGe49xW2cl0iZ4Fqu9xhCS1hlSLDNNN1Azw1hwptGkP
-3voTVWPzU95EXhqtnDhf7QO4QpcIZ44Zm0pWMoyip9TE16bTwjltiAdOs2shK+9h
-oQ9b2OW69cWpvtwnWsWAACveQHFUZ7drIDc7cv3OPvo7aonx4CHKqxEqnmlIxipP
-t8nkKtyHr10CAwEAAaOBnDCBmTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYI
-KwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFPc/GrBA
-B2mcJnSACyKgkNt5GWvrMB8GA1UdIwQYMBaAFN1lMCehxjzmUeHJH8DuUmSne3aa
+ZWN1cml0eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALKKf9zhwcgu
+S53Vd92PLqHtDNpTbl4nM/5qYy7nHV5FHW7uolQHFA0BjJwxz7/LC39G950PpSIs
+K7Y99P7aMPVqVH5FGL+uoDbHMzt/gIUGgWj51J38+x6zN/9vIvAVIBClBhzEuB2k
+WJU6KyB1V5G+1wnnKRXLB9QC0f/7vqd9f21O7sJmWeVhGJuEUwwAp1p5WDGM2Tn8
+Fjy57O9f1nT4WVqWhB5EbvYGDF2Z5DWyKz90EWOwVw30ThcQHF57X6WJdlNiQOrY
+KIWO475QKPwbUSpRkvw1jwvllU8s4l6pB1hKTAcUK3UsWrzpf2+m8v8ou35uFD/c
+pd2bBx5MVlkCAwEAAaOBnDCBmTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYI
+KwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFAcMdmvX
+8NC4V4Z67Jb/pNUwai7cMB8GA1UdIwQYMBaAFGUHvq+NpnIUusRQaboNB/KOY3o+
 MBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0BAQsFAAOCAQEA
-F30eIP+5u2gZItB9VpohOFjKoW2Ts+TIGT+nE2/b70kpX1JXuOHvJSFgYxn1N9KK
-tJ03T52ENwhFw7ES9sYmpAKjt/Z+2W1DnY1n29fWvGYKReOt3TjjPQcC26v9l+Kf
-oNSFQEac2rdzKZEzkPDRpvlwVQuACGNbLpbpFmV4G1lavt0ecRuhohHgCtO03nx+
-UOQ1Enla3gA002qw2OI71cmrxYzpS3tgZCGR3QsOOibls+/1kBQE/F4HIyhLIImo
-dGC98Q1hLYVS8oKwK+DifK4mDdRSbXE7ek/5+vMOsKWdcs2dTcn9WoijJV6YVX/e
-HU7ocaMm/c5SQ0yndcCU2w==
+btQjJEWHD/0gYsNLFg3tDxZ64U/HfNlh4USGOK02VL2LteMcV8AoYlZ3jwmp4+33
+D3HlqLclJNABax2pOvTHVnQlf25TSNwJRtmzOvcg+6xYbPdgRoeEVsWbmgbpX7Vi
+P8FYelYCiYTPezjqZgPG1gmq0Uf/drlTrjwsG2njEcuK7hip+LdJnIrtpIrabpIk
+lZRa7Y/JBM3gP/rR1fu9lhzJ97s3NabuHzPwyouSTTknaaiGwSV8F5frh9NGcFhd
+G7giCLZLKklQB4IUTOFcVFSZmeAGy6KBqyT10N2kkBrsrcWhyMKIU9X0+6hh3Tlc
+JEla9as6qFvt1dFGp+qeww==
 -----END CERTIFICATE-----

integration/fixtures/server3.key.insecure

@@ -1,27 +1,27 @@
 -----BEGIN RSA PRIVATE KEY-----
-MIIEpAIBAAKCAQEAt/0rMGjckVnVmLyUfi3oxILH917UzIdJHhLSwP6uX8Zx2R08
-s4sYqKQdksxTln27XLkcrbdBYHIpx0sUORByShQEPEBH6zF1VflGBVOQxH3N2//g
-dV6u1qQo262uejSVR2qEK1XTlhEd5Tz+08xyLIThwFt5IuW/EZ7j3FbZyXSJngWq
-73GEJLWGVIsM003UDPDWHCm0aQ/e+hNVY/NT3kReGq2cOF/tA7hClwhnjhmbSlYy
-jKKn1MTXptPCOW2IB06zayEr72GhD1vY5br1xam+3CdaxYAAK95AcVRnt2sgNzty
-/c4++jtqifHgIcqrESqeaUjGKk+3yeQq3IevXQIDAQABAoIBACc8fjFcq2zz03E3
-BhPChpkhqrM+LfNQBm+7Y+Z0aYtTLoQ2j/NZ/nA4T3Y2zLyTI5mqgEsrgW2n6vDt
-OerRbw4NJroSm0O00Gj9N8l7AKxmh7ga4SsmffKYH355k3PMMul/9Z+oNe8Sx2VT
-lKRmPRLxY5M57qNai1yencknXFpxetSzHwusfDFFNiStN8rTDeIgJtn9FKxhMAcJ
-P0D8rHH+x4/8XGSVtjANJr22cj1win0dce4tVVUQp60pBAFhGIY+uPOM8vmxkJYx
-xQTmR6bh7S5KrYT1BNK3cpo6gQGE6HJsPzTuijN6hqvJ7ufTl/lq2T0ETVEemHOf
-B5NKOwECgYEAxigkRZeaCsvvUcPWEYrnIhZt6SfCx5iKlkVQsG/uYps3aWohkBAY
-Ws6rr55qkH7hsye9ikeyhT9VDnWHeAbTr2SKIsa8CAc+gQoPzYfY4GVYzbNP+/rj
-d1jS/s5/LyVryi3ln3EJN1fUHv+ZzvEWORSa/WVHkBuxMxoeHX8PhSkCgYEA7bJK
-inBlLXtUy+c/q5DigYLIg0MquQ9slcfJapcwgswmbLnagtW9oQxlsM/uSiqR4br+
-9SKD7TNLNhwJmByG6HlPyjCF2FGPhUg4pIWuRckPqJ6o2SgONupD3mZdu/CyzsuO
-pyCqq0d2yeLXW77MxidMUeNfPsAA9lvRRulhCxUCgYEAwbM1ma2TD+DABP5ZQHa2
-b3TbZeHPHgr31eLV+FLCBTPTG8F6I3gIRqPl4dsKMktFVzqOpiBl2qjI/URX8zVB
-Mh8mhM4duf9S0xLB1dhoYRnQj+srUZazSdPTFO9IFg8PaegpoQz+xFGfcdnLQSYb
-4hpJU0/wf2cCdYCfVZgB1NkCgYB9K/a1EJs3aEsvVYfiAVpGeWi+NxC4g7ba6WrY
-BuY0+u0BNJ4taAGEXdLvWZBS3jgUdzTsQlDXCLwCsqEayWsB4WBzSToywECkH3Q0
-r3EmrsrgMS0Zrk5N/O/gnmeeIRMIc4mb2UgHCoszpZFjbwbHEsrOFL6DfPkEwzVh
-8mR4QQKBgQC88WRxSGnF0AyoM5tB8/x80O9a4CLsSwfFVzEyJcdh3uL9+JzfNVta
-YsXy5ah2SyBrI7xQd/LXhv3/7GY4eZIQuwcOH74hQNHdm2u3bWfmg39jH+HgWK8m
-mqYvWYE9qLXIVwBzsCzQtxq4z90KMiZ2qXV0YMFPGjZt+bBkbjJTOw==
+MIIEpAIBAAKCAQEAsop/3OHByC5LndV33Y8uoe0M2lNuXicz/mpjLucdXkUdbu6i
+VAcUDQGMnDHPv8sLf0b3nQ+lIiwrtj30/tow9WpUfkUYv66gNsczO3+AhQaBaPnU
+nfz7HrM3/28i8BUgEKUGHMS4HaRYlTorIHVXkb7XCecpFcsH1ALR//u+p31/bU7u
+wmZZ5WEYm4RTDACnWnlYMYzZOfwWPLns71/WdPhZWpaEHkRu9gYMXZnkNbIrP3QR
+Y7BXDfROFxAcXntfpYl2U2JA6tgohY7jvlAo/BtRKlGS/DWPC+WVTyziXqkHWEpM
+BxQrdSxavOl/b6by/yi7fm4UP9yl3ZsHHkxWWQIDAQABAoIBAQCVyfrCDqlsT+Li
+1UBOIp0l/uIEnXCAD3XgodL6e6249FVgR1brFlEtJDqapHO+XhQUQS7ml0ScqeA2
+cj6EPfxLOV0P3tqHnnMN4gvKhAsID9AsiUVnEuJ//C4j4FK4h5CyRjEdm7E4NTSY
+ZgfeoHPKdAinZ0eh4Ad+SKt0jvmCPDD1L+6bxpJ6E258xPDxB71rHTCgnwZZmXrN
+rHDg07tVVU6lYtXEsZAsyIBIxXV/RaCt4xSijM1C7kuSsUE3+CCw+pzQUvHDxkuk
+FPxE5hONzkUaCSBKTv4L6gVaiYa30Jo9THuTRWvzDJmcnNwlYgRLeIR8PJY0eHqv
+FYJbLdQBAoGBAOeiW1aHpNO+K4BqVMzm7fvz0a3DBkml/wIKSvw3dbTYnyIXFvDB
+Be7OZLhiWPwaE/58aQWh+/OGCr88yyCLAt2mOQ5aMalKpdSCaY5swEnzt923pY9z
+jt3DnhX8aXggPbqM6eJjaxJY7jIMSDNKUQZKJeSesr/EzFo0CRkkepyBAoGBAMVS
+Z2nIY+G2+P1VSUEbI3dbaZ3ciMEDLc+rZQs0fx8+xQCD71eCU9ggxf+O0R1/0smm
+9so65KrmKl3yOt8OQW3YgUpQqIQdJPHnfuTnsU7y/+zRl6k119gV89LLDlCM7nfW
+5/ey/iJLXQxfC7OoFF/hQM0odmorA8jBuqKDWy3ZAoGAZL6gq0njzpRvpzKYH2Zx
+K5woHkMsgOvJtcF0S65za2ysCc+xEpVhVzQ9alScD0noWE8T/nctdgVetz5huo27
+eVvKhQuFffQRnBP8hQ2XtJJj7fLp9zJzeNCT+UwHM1ASiQiw0N4cu6YiM3JUFLrF
+8s5dHMpJRE778l+fdWgATAECgYEAmJ3osE+2uUCtCjvpwbp8zvdcFCYbe7W6vBGj
+wGvlGsSQ2JozB2sc8GBA5C2RHhDcdu11meq9LFWDVVBiKl27S3uWXGVQQYbNKXDU
+m7V8VUTrnz5o4A5uGIq6IEK/mpu2YehNWC8QEnRZzpTA1z7cK2Bsn4F5PRpx/deh
+Q8r3PdkCgYBd2rT+S4/51C5AnIhgMF/PYl0+DYMFD8HAfsx7VTaIUmFQ4devHMOz
+J6lbqRyEITZtXgna1n35LkyBDcPwsEtntjJOP+xneCtKzozdzhXyoAw2xQR3gqvV
+7YtV3miYQiOTqjOefViMhR/XiOV2zng3OId1AQObfOUZODJPfSN26g==
 -----END RSA PRIVATE KEY-----

integration/v3_alarm_test.go

@@ -27,6 +27,7 @@ import (
 	"go.etcd.io/etcd/mvcc"
 	"go.etcd.io/etcd/mvcc/backend"
 	"go.etcd.io/etcd/pkg/testutil"
+	"go.etcd.io/etcd/pkg/traceutil"
 
 	"go.uber.org/zap"
 )
@@ -173,7 +174,7 @@ func TestV3CorruptAlarm(t *testing.T) {
 	// NOTE: cluster_proxy mode with namespacing won't set 'k', but namespace/'k'.
 	s.Put([]byte("abc"), []byte("def"), 0)
 	s.Put([]byte("xyz"), []byte("123"), 0)
-	s.Compact(5)
+	s.Compact(traceutil.TODO(), 5)
 	s.Commit()
 	s.Close()
 	be.Close()

integration/v3_lock_test.go

@@ -23,30 +23,30 @@ import (
 
 	"go.etcd.io/etcd/clientv3"
 	"go.etcd.io/etcd/clientv3/concurrency"
-	"go.etcd.io/etcd/contrib/recipes"
+	recipe "go.etcd.io/etcd/contrib/recipes"
 	"go.etcd.io/etcd/mvcc/mvccpb"
 	"go.etcd.io/etcd/pkg/testutil"
 )
 
-func TestMutexSingleNode(t *testing.T) {
+func TestMutexLockSingleNode(t *testing.T) {
 	clus := NewClusterV3(t, &ClusterConfig{Size: 3})
 	defer clus.Terminate(t)
 
 	var clients []*clientv3.Client
-	testMutex(t, 5, makeSingleNodeClients(t, clus.cluster, &clients))
+	testMutexLock(t, 5, makeSingleNodeClients(t, clus.cluster, &clients))
 	closeClients(t, clients)
 }
 
-func TestMutexMultiNode(t *testing.T) {
+func TestMutexLockMultiNode(t *testing.T) {
 	clus := NewClusterV3(t, &ClusterConfig{Size: 3})
 	defer clus.Terminate(t)
 
 	var clients []*clientv3.Client
-	testMutex(t, 5, makeMultiNodeClients(t, clus.cluster, &clients))
+	testMutexLock(t, 5, makeMultiNodeClients(t, clus.cluster, &clients))
 	closeClients(t, clients)
 }
 
-func testMutex(t *testing.T, waiters int, chooseClient func() *clientv3.Client) {
+func testMutexLock(t *testing.T, waiters int, chooseClient func() *clientv3.Client) {
 	// stream lock acquisitions
 	lockedC := make(chan *concurrency.Mutex)
 	for i := 0; i < waiters; i++ {
@@ -82,6 +82,62 @@ func testMutex(t *testing.T, waiters int, chooseClient func() *clientv3.Client)
 	}
 }
 
+func TestMutexTryLockSingleNode(t *testing.T) {
+	clus := NewClusterV3(t, &ClusterConfig{Size: 3})
+	defer clus.Terminate(t)
+
+	var clients []*clientv3.Client
+	testMutexTryLock(t, 5, makeSingleNodeClients(t, clus.cluster, &clients))
+	closeClients(t, clients)
+}
+
+func TestMutexTryLockMultiNode(t *testing.T) {
+	clus := NewClusterV3(t, &ClusterConfig{Size: 3})
+	defer clus.Terminate(t)
+
+	var clients []*clientv3.Client
+	testMutexTryLock(t, 5, makeMultiNodeClients(t, clus.cluster, &clients))
+	closeClients(t, clients)
+}
+
+func testMutexTryLock(t *testing.T, lockers int, chooseClient func() *clientv3.Client) {
+	lockedC := make(chan *concurrency.Mutex)
+	notlockedC := make(chan *concurrency.Mutex)
+	for i := 0; i < lockers; i++ {
+		go func() {
+			session, err := concurrency.NewSession(chooseClient())
+			if err != nil {
+				t.Error(err)
+			}
+			m := concurrency.NewMutex(session, "test-mutex-try-lock")
+			err = m.TryLock(context.TODO())
+			if err == nil {
+				lockedC <- m
+			} else if err == concurrency.ErrLocked {
+				notlockedC <- m
+			} else {
+				t.Errorf("Unexpected Error %v", err)
+			}
+		}()
+	}
+
+	timerC := time.After(time.Second)
+	select {
+	case <-lockedC:
+		for i := 0; i < lockers-1; i++ {
+			select {
+			case <-lockedC:
+				t.Fatalf("Multiple Mutes locked on same key")
+			case <-notlockedC:
+			case <-timerC:
+				t.Errorf("timed out waiting for lock")
+			}
+		}
+	case <-timerC:
+		t.Errorf("timed out waiting for lock")
+	}
+}
+
 // TestMutexSessionRelock ensures that acquiring the same lock with the same
 // session will not result in deadlock.
 func TestMutexSessionRelock(t *testing.T) {
@@ -219,7 +275,7 @@ func BenchmarkMutex4Waiters(b *testing.B) {
 	clus := NewClusterV3(nil, &ClusterConfig{Size: 3})
 	defer clus.Terminate(nil)
 	for i := 0; i < b.N; i++ {
-		testMutex(nil, 4, func() *clientv3.Client { return clus.RandClient() })
+		testMutexLock(nil, 4, func() *clientv3.Client { return clus.RandClient() })
 	}
 }
 

mvcc/kv.go

@@ -18,6 +18,7 @@ import (
 	"go.etcd.io/etcd/lease"
 	"go.etcd.io/etcd/mvcc/backend"
 	"go.etcd.io/etcd/mvcc/mvccpb"
+	"go.etcd.io/etcd/pkg/traceutil"
 )
 
 type RangeOptions struct {
@@ -102,10 +103,10 @@ type KV interface {
 	WriteView
 
 	// Read creates a read transaction.
-	Read() TxnRead
+	Read(trace *traceutil.Trace) TxnRead
 
 	// Write creates a write transaction.
-	Write() TxnWrite
+	Write(trace *traceutil.Trace) TxnWrite
 
 	// Hash computes the hash of the KV's backend.
 	Hash() (hash uint32, revision int64, err error)
@@ -114,7 +115,7 @@ type KV interface {
 	HashByRev(rev int64) (hash uint32, revision int64, compactRev int64, err error)
 
 	// Compact frees all superseded keys with revisions less than rev.
-	Compact(rev int64) (<-chan struct{}, error)
+	Compact(trace *traceutil.Trace, rev int64) (<-chan struct{}, error)
 
 	// Commit commits outstanding txns into the underlying backend.
 	Commit()

mvcc/kv_test.go

@@ -25,6 +25,7 @@ import (
 	"go.etcd.io/etcd/mvcc/backend"
 	"go.etcd.io/etcd/mvcc/mvccpb"
 	"go.etcd.io/etcd/pkg/testutil"
+	"go.etcd.io/etcd/pkg/traceutil"
 
 	"github.com/prometheus/client_golang/prometheus"
 	dto "github.com/prometheus/client_model/go"
@@ -47,7 +48,7 @@ var (
 		return kv.Range(key, end, ro)
 	}
 	txnRangeFunc = func(kv KV, key, end []byte, ro RangeOptions) (*RangeResult, error) {
-		txn := kv.Read()
+		txn := kv.Read(traceutil.TODO())
 		defer txn.End()
 		return txn.Range(key, end, ro)
 	}
@@ -56,7 +57,7 @@ var (
 		return kv.Put(key, value, lease)
 	}
 	txnPutFunc = func(kv KV, key, value []byte, lease lease.LeaseID) int64 {
-		txn := kv.Write()
+		txn := kv.Write(traceutil.TODO())
 		defer txn.End()
 		return txn.Put(key, value, lease)
 	}
@@ -65,7 +66,7 @@ var (
 		return kv.DeleteRange(key, end)
 	}
 	txnDeleteRangeFunc = func(kv KV, key, end []byte) (n, rev int64) {
-		txn := kv.Write()
+		txn := kv.Write(traceutil.TODO())
 		defer txn.End()
 		return txn.DeleteRange(key, end)
 	}
@@ -182,7 +183,7 @@ func testKVRangeBadRev(t *testing.T, f rangeFunc) {
 	defer cleanup(s, b, tmpPath)
 
 	put3TestKVs(s)
-	if _, err := s.Compact(4); err != nil {
+	if _, err := s.Compact(traceutil.TODO(), 4); err != nil {
 		t.Fatalf("compact error (%v)", err)
 	}
 
@@ -409,7 +410,7 @@ func TestKVTxnBlockWriteOperations(t *testing.T) {
 		func() { s.DeleteRange([]byte("foo"), nil) },
 	}
 	for i, tt := range tests {
-		txn := s.Write()
+		txn := s.Write(traceutil.TODO())
 		done := make(chan struct{}, 1)
 		go func() {
 			tt()
@@ -438,7 +439,7 @@ func TestKVTxnNonBlockRange(t *testing.T) {
 	s := NewStore(zap.NewExample(), b, &lease.FakeLessor{}, nil, StoreConfig{})
 	defer cleanup(s, b, tmpPath)
 
-	txn := s.Write()
+	txn := s.Write(traceutil.TODO())
 	defer txn.End()
 
 	donec := make(chan struct{})
@@ -460,7 +461,7 @@ func TestKVTxnOperationInSequence(t *testing.T) {
 	defer cleanup(s, b, tmpPath)
 
 	for i := 0; i < 10; i++ {
-		txn := s.Write()
+		txn := s.Write(traceutil.TODO())
 		base := int64(i + 1)
 
 		// put foo
@@ -544,7 +545,7 @@ func TestKVCompactReserveLastValue(t *testing.T) {
 		},
 	}
 	for i, tt := range tests {
-		_, err := s.Compact(tt.rev)
+		_, err := s.Compact(traceutil.TODO(), tt.rev)
 		if err != nil {
 			t.Errorf("#%d: unexpect compact error %v", i, err)
 		}
@@ -580,7 +581,7 @@ func TestKVCompactBad(t *testing.T) {
 		{100, ErrFutureRev},
 	}
 	for i, tt := range tests {
-		_, err := s.Compact(tt.rev)
+		_, err := s.Compact(traceutil.TODO(), tt.rev)
 		if err != tt.werr {
 			t.Errorf("#%d: compact error = %v, want %v", i, err, tt.werr)
 		}
@@ -626,7 +627,7 @@ func TestKVRestore(t *testing.T) {
 		func(kv KV) {
 			kv.Put([]byte("foo"), []byte("bar0"), 1)
 			kv.Put([]byte("foo"), []byte("bar1"), 2)
-			kv.Compact(1)
+			kv.Compact(traceutil.TODO(), 1)
 		},
 	}
 	for i, tt := range tests {

mvcc/kv_view.go

@@ -14,24 +14,27 @@
 
 package mvcc
 
-import "go.etcd.io/etcd/lease"
+import (
+	"go.etcd.io/etcd/lease"
+	"go.etcd.io/etcd/pkg/traceutil"
+)
 
 type readView struct{ kv KV }
 
 func (rv *readView) FirstRev() int64 {
-	tr := rv.kv.Read()
+	tr := rv.kv.Read(traceutil.TODO())
 	defer tr.End()
 	return tr.FirstRev()
 }
 
 func (rv *readView) Rev() int64 {
-	tr := rv.kv.Read()
+	tr := rv.kv.Read(traceutil.TODO())
 	defer tr.End()
 	return tr.Rev()
 }
 
 func (rv *readView) Range(key, end []byte, ro RangeOptions) (r *RangeResult, err error) {
-	tr := rv.kv.Read()
+	tr := rv.kv.Read(traceutil.TODO())
 	defer tr.End()
 	return tr.Range(key, end, ro)
 }
@@ -39,13 +42,13 @@ func (rv *readView) Range(key, end []byte, ro RangeOptions) (r *RangeResult, err
 type writeView struct{ kv KV }
 
 func (wv *writeView) DeleteRange(key, end []byte) (n, rev int64) {
-	tw := wv.kv.Write()
+	tw := wv.kv.Write(traceutil.TODO())
 	defer tw.End()
 	return tw.DeleteRange(key, end)
 }
 
 func (wv *writeView) Put(key, value []byte, lease lease.LeaseID) (rev int64) {
-	tw := wv.kv.Write()
+	tw := wv.kv.Write(traceutil.TODO())
 	defer tw.End()
 	return tw.Put(key, value, lease)
 }

mvcc/kvstore.go

@@ -29,6 +29,7 @@ import (
 	"go.etcd.io/etcd/mvcc/backend"
 	"go.etcd.io/etcd/mvcc/mvccpb"
 	"go.etcd.io/etcd/pkg/schedule"
+	"go.etcd.io/etcd/pkg/traceutil"
 
 	"github.com/coreos/pkg/capnslog"
 	"go.uber.org/zap"
@@ -140,7 +141,7 @@ func NewStore(lg *zap.Logger, b backend.Backend, le lease.Lessor, ig ConsistentI
 	s.ReadView = &readView{s}
 	s.WriteView = &writeView{s}
 	if s.le != nil {
-		s.le.SetRangeDeleter(func() lease.TxnDelete { return s.Write() })
+		s.le.SetRangeDeleter(func() lease.TxnDelete { return s.Write(traceutil.TODO()) })
 	}
 
 	tx := s.b.BatchTx()
@@ -270,9 +271,10 @@ func (s *store) updateCompactRev(rev int64) (<-chan struct{}, error) {
 	return nil, nil
 }
 
-func (s *store) compact(rev int64) (<-chan struct{}, error) {
+func (s *store) compact(trace *traceutil.Trace, rev int64) (<-chan struct{}, error) {
 	start := time.Now()
 	keep := s.kvindex.Compact(rev)
+	trace.Step("compact in-memory index tree")
 	ch := make(chan struct{})
 	var j = func(ctx context.Context) {
 		if ctx.Err() != nil {
@@ -289,6 +291,7 @@ func (s *store) compact(rev int64) (<-chan struct{}, error) {
 	s.fifoSched.Schedule(j)
 
 	indexCompactionPauseMs.Observe(float64(time.Since(start) / time.Millisecond))
+	trace.Step("schedule compaction")
 	return ch, nil
 }
 
@@ -298,21 +301,21 @@ func (s *store) compactLockfree(rev int64) (<-chan struct{}, error) {
 		return ch, err
 	}
 
-	return s.compact(rev)
+	return s.compact(traceutil.TODO(), rev)
 }
 
-func (s *store) Compact(rev int64) (<-chan struct{}, error) {
+func (s *store) Compact(trace *traceutil.Trace, rev int64) (<-chan struct{}, error) {
 	s.mu.Lock()
 
 	ch, err := s.updateCompactRev(rev)
-
+	trace.Step("check and update compact revision")
 	if err != nil {
 		s.mu.Unlock()
 		return ch, err
 	}
 	s.mu.Unlock()
 
-	return s.compact(rev)
+	return s.compact(trace, rev)
 }
 
 // DefaultIgnores is a map of keys to ignore in hash checking.

mvcc/kvstore_bench_test.go

@@ -20,6 +20,7 @@ import (
 
 	"go.etcd.io/etcd/lease"
 	"go.etcd.io/etcd/mvcc/backend"
+	"go.etcd.io/etcd/pkg/traceutil"
 
 	"go.uber.org/zap"
 )
@@ -130,7 +131,7 @@ func BenchmarkStoreTxnPut(b *testing.B) {
 	b.ResetTimer()
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		txn := s.Write()
+		txn := s.Write(traceutil.TODO())
 		txn.Put(keys[i], vals[i], lease.NoLease)
 		txn.End()
 	}
@@ -151,7 +152,7 @@ func benchmarkStoreRestore(revsPerKey int, b *testing.B) {
 
 	for i := 0; i < b.N; i++ {
 		for j := 0; j < revsPerKey; j++ {
-			txn := s.Write()
+			txn := s.Write(traceutil.TODO())
 			txn.Put(keys[i], vals[i], lease.NoLease)
 			txn.End()
 		}

mvcc/kvstore_compaction_test.go

@@ -22,6 +22,7 @@ import (
 
 	"go.etcd.io/etcd/lease"
 	"go.etcd.io/etcd/mvcc/backend"
+	"go.etcd.io/etcd/pkg/traceutil"
 	"go.uber.org/zap"
 )
 
@@ -109,7 +110,7 @@ func TestCompactAllAndRestore(t *testing.T) {
 
 	rev := s0.Rev()
 	// compact all keys
-	done, err := s0.Compact(rev)
+	done, err := s0.Compact(traceutil.TODO(), rev)
 	if err != nil {
 		t.Fatal(err)
 	}

mvcc/kvstore_test.go

@@ -34,6 +34,7 @@ import (
 	"go.etcd.io/etcd/mvcc/mvccpb"
 	"go.etcd.io/etcd/pkg/schedule"
 	"go.etcd.io/etcd/pkg/testutil"
+	"go.etcd.io/etcd/pkg/traceutil"
 
 	"go.uber.org/zap"
 )
@@ -331,7 +332,7 @@ func TestStoreCompact(t *testing.T) {
 	key2 := newTestKeyBytes(revision{2, 0}, false)
 	b.tx.rangeRespc <- rangeResp{[][]byte{key1, key2}, nil}
 
-	s.Compact(3)
+	s.Compact(traceutil.TODO(), 3)
 	s.fifoSched.WaitFinish(1)
 
 	if s.compactMainRev != 3 {
@@ -582,7 +583,7 @@ func TestHashKVWhenCompacting(t *testing.T) {
 	go func() {
 		defer wg.Done()
 		for i := 100; i >= 0; i-- {
-			_, err := s.Compact(int64(rev - 1 - i))
+			_, err := s.Compact(traceutil.TODO(), int64(rev-1-i))
 			if err != nil {
 				t.Error(err)
 			}
@@ -609,7 +610,7 @@ func TestHashKVZeroRevision(t *testing.T) {
 	for i := 2; i <= rev; i++ {
 		s.Put([]byte("foo"), []byte(fmt.Sprintf("bar%d", i)), lease.NoLease)
 	}
-	if _, err := s.Compact(int64(rev / 2)); err != nil {
+	if _, err := s.Compact(traceutil.TODO(), int64(rev/2)); err != nil {
 		t.Fatal(err)
 	}
 
@@ -639,7 +640,7 @@ func TestTxnPut(t *testing.T) {
 	defer cleanup(s, b, tmpPath)
 
 	for i := 0; i < sliceN; i++ {
-		txn := s.Write()
+		txn := s.Write(traceutil.TODO())
 		base := int64(i + 2)
 		if rev := txn.Put(keys[i], vals[i], lease.NoLease); rev != base {
 			t.Errorf("#%d: rev = %d, want %d", i, rev, base)
@@ -658,7 +659,7 @@ func TestConcurrentReadNotBlockingWrite(t *testing.T) {
 	s.Put([]byte("foo"), []byte("bar"), lease.NoLease)
 
 	// readTx simulates a long read request
-	readTx1 := s.Read()
+	readTx1 := s.Read(traceutil.TODO())
 
 	// write should not be blocked by reads
 	done := make(chan struct{})
@@ -673,7 +674,7 @@ func TestConcurrentReadNotBlockingWrite(t *testing.T) {
 	}
 
 	// readTx2 simulates a short read request
-	readTx2 := s.Read()
+	readTx2 := s.Read(traceutil.TODO())
 	ro := RangeOptions{Limit: 1, Rev: 0, Count: false}
 	ret, err := readTx2.Range([]byte("foo"), nil, ro)
 	if err != nil {
@@ -730,7 +731,7 @@ func TestConcurrentReadTxAndWrite(t *testing.T) {
 			defer wg.Done()
 			time.Sleep(time.Duration(mrand.Intn(100)) * time.Millisecond) // random starting time
 
-			tx := s.Write()
+			tx := s.Write(traceutil.TODO())
 			numOfPuts := mrand.Intn(maxNumOfPutsPerWrite) + 1
 			var pendingKvs kvs
 			for j := 0; j < numOfPuts; j++ {
@@ -756,7 +757,7 @@ func TestConcurrentReadTxAndWrite(t *testing.T) {
 			mu.Lock()
 			wKVs := make(kvs, len(committedKVs))
 			copy(wKVs, committedKVs)
-			tx := s.Read()
+			tx := s.Read(traceutil.TODO())
 			mu.Unlock()
 			// get all keys in backend store, and compare with wKVs
 			ret, err := tx.Range([]byte("\x00000000"), []byte("\xffffffff"), RangeOptions{})

mvcc/kvstore_txn.go

@@ -18,6 +18,7 @@ import (
 	"go.etcd.io/etcd/lease"
 	"go.etcd.io/etcd/mvcc/backend"
 	"go.etcd.io/etcd/mvcc/mvccpb"
+	"go.etcd.io/etcd/pkg/traceutil"
 	"go.uber.org/zap"
 )
 
@@ -27,9 +28,11 @@ type storeTxnRead struct {
 
 	firstRev int64
 	rev      int64
+
+	trace *traceutil.Trace
 }
 
-func (s *store) Read() TxnRead {
+func (s *store) Read(trace *traceutil.Trace) TxnRead {
 	s.mu.RLock()
 	s.revMu.RLock()
 	// backend holds b.readTx.RLock() only when creating the concurrentReadTx. After
@@ -38,7 +41,7 @@ func (s *store) Read() TxnRead {
 	tx.RLock() // RLock is no-op. concurrentReadTx does not need to be locked after it is created.
 	firstRev, rev := s.compactMainRev, s.currentRev
 	s.revMu.RUnlock()
-	return newMetricsTxnRead(&storeTxnRead{s, tx, firstRev, rev})
+	return newMetricsTxnRead(&storeTxnRead{s, tx, firstRev, rev, trace})
 }
 
 func (tr *storeTxnRead) FirstRev() int64 { return tr.firstRev }
@@ -61,12 +64,12 @@ type storeTxnWrite struct {
 	changes  []mvccpb.KeyValue
 }
 
-func (s *store) Write() TxnWrite {
+func (s *store) Write(trace *traceutil.Trace) TxnWrite {
 	s.mu.RLock()
 	tx := s.b.BatchTx()
 	tx.Lock()
 	tw := &storeTxnWrite{
-		storeTxnRead: storeTxnRead{s, tx, 0, 0},
+		storeTxnRead: storeTxnRead{s, tx, 0, 0, trace},
 		tx:           tx,
 		beginRev:     s.currentRev,
 		changes:      make([]mvccpb.KeyValue, 0, 4),
@@ -124,6 +127,7 @@ func (tr *storeTxnRead) rangeKeys(key, end []byte, curRev int64, ro RangeOptions
 	}
 
 	revpairs := tr.s.kvindex.Revisions(key, end, rev)
+	tr.trace.Step("range keys from in-memory index tree")
 	if len(revpairs) == 0 {
 		return &RangeResult{KVs: nil, Count: 0, Rev: curRev}, nil
 	}
@@ -163,6 +167,7 @@ func (tr *storeTxnRead) rangeKeys(key, end []byte, curRev int64, ro RangeOptions
 			}
 		}
 	}
+	tr.trace.Step("range keys from bolt db")
 	return &RangeResult{KVs: kvs, Count: len(revpairs), Rev: curRev}, nil
 }
 
@@ -178,7 +183,7 @@ func (tw *storeTxnWrite) put(key, value []byte, leaseID lease.LeaseID) {
 		c = created.main
 		oldLease = tw.s.le.GetLease(lease.LeaseItem{Key: string(key)})
 	}
-
+	tw.trace.Step("get key's previous created_revision and leaseID")
 	ibytes := newRevBytes()
 	idxRev := revision{main: rev, sub: int64(len(tw.changes))}
 	revToBytes(idxRev, ibytes)
@@ -205,9 +210,11 @@ func (tw *storeTxnWrite) put(key, value []byte, leaseID lease.LeaseID) {
 		}
 	}
 
+	tw.trace.Step("marshal mvccpb.KeyValue")
 	tw.tx.UnsafeSeqPut(keyBucketName, ibytes, d)
 	tw.s.kvindex.Put(key, idxRev)
 	tw.changes = append(tw.changes, kv)
+	tw.trace.Step("store kv pair into bolt db")
 
 	if oldLease != lease.NoLease {
 		if tw.s.le == nil {
@@ -234,6 +241,7 @@ func (tw *storeTxnWrite) put(key, value []byte, leaseID lease.LeaseID) {
 			panic("unexpected error from lease Attach")
 		}
 	}
+	tw.trace.Step("attach lease to kv pair")
 }
 
 func (tw *storeTxnWrite) deleteRange(key, end []byte) int64 {

mvcc/watchable_store.go

@@ -21,6 +21,7 @@ import (
 	"go.etcd.io/etcd/lease"
 	"go.etcd.io/etcd/mvcc/backend"
 	"go.etcd.io/etcd/mvcc/mvccpb"
+	"go.etcd.io/etcd/pkg/traceutil"
 	"go.uber.org/zap"
 )
 
@@ -84,7 +85,7 @@ func newWatchableStore(lg *zap.Logger, b backend.Backend, le lease.Lessor, ig Co
 	s.store.WriteView = &writeView{s}
 	if s.le != nil {
 		// use this store as the deleter so revokes trigger watch events
-		s.le.SetRangeDeleter(func() lease.TxnDelete { return s.Write() })
+		s.le.SetRangeDeleter(func() lease.TxnDelete { return s.Write(traceutil.TODO()) })
 	}
 	s.wg.Add(2)
 	go s.syncWatchersLoop()

mvcc/watchable_store_bench_test.go

@@ -21,6 +21,7 @@ import (
 
 	"go.etcd.io/etcd/lease"
 	"go.etcd.io/etcd/mvcc/backend"
+	"go.etcd.io/etcd/pkg/traceutil"
 
 	"go.uber.org/zap"
 )
@@ -59,7 +60,7 @@ func BenchmarkWatchableStoreTxnPut(b *testing.B) {
 	b.ResetTimer()
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		txn := s.Write()
+		txn := s.Write(traceutil.TODO())
 		txn.Put(keys[i], vals[i], lease.NoLease)
 		txn.End()
 	}

mvcc/watchable_store_test.go

@@ -26,6 +26,7 @@ import (
 	"go.etcd.io/etcd/lease"
 	"go.etcd.io/etcd/mvcc/backend"
 	"go.etcd.io/etcd/mvcc/mvccpb"
+	"go.etcd.io/etcd/pkg/traceutil"
 	"go.uber.org/zap"
 )
 
@@ -237,7 +238,7 @@ func TestWatchCompacted(t *testing.T) {
 	for i := 0; i < maxRev; i++ {
 		s.Put(testKey, testValue, lease.NoLease)
 	}
-	_, err := s.Compact(compactRev)
+	_, err := s.Compact(traceutil.TODO(), compactRev)
 	if err != nil {
 		t.Fatalf("failed to compact kv (%v)", err)
 	}

mvcc/watchable_store_txn.go

@@ -14,7 +14,10 @@
 
 package mvcc
 
-import "go.etcd.io/etcd/mvcc/mvccpb"
+import (
+	"go.etcd.io/etcd/mvcc/mvccpb"
+	"go.etcd.io/etcd/pkg/traceutil"
+)
 
 func (tw *watchableStoreTxnWrite) End() {
 	changes := tw.Changes()
@@ -48,4 +51,6 @@ type watchableStoreTxnWrite struct {
 	s *watchableStore
 }
 
-func (s *watchableStore) Write() TxnWrite { return &watchableStoreTxnWrite{s.store.Write(), s} }
+func (s *watchableStore) Write(trace *traceutil.Trace) TxnWrite {
+	return &watchableStoreTxnWrite{s.store.Write(trace), s}
+}

pkg/traceutil/trace.go

@@ -0,0 +1,172 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package traceutil implements tracing utilities using "context".
+package traceutil
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"math/rand"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+const (
+	TraceKey     = "trace"
+	StartTimeKey = "startTime"
+)
+
+// Field is a kv pair to record additional details of the trace.
+type Field struct {
+	Key   string
+	Value interface{}
+}
+
+func (f *Field) format() string {
+	return fmt.Sprintf("%s:%v; ", f.Key, f.Value)
+}
+
+func writeFields(fields []Field) string {
+	if len(fields) == 0 {
+		return ""
+	}
+	var buf bytes.Buffer
+	buf.WriteString("{")
+	for _, f := range fields {
+		buf.WriteString(f.format())
+	}
+	buf.WriteString("}")
+	return buf.String()
+}
+
+type Trace struct {
+	operation    string
+	lg           *zap.Logger
+	fields       []Field
+	startTime    time.Time
+	steps        []step
+	stepDisabled bool
+}
+
+type step struct {
+	time   time.Time
+	msg    string
+	fields []Field
+}
+
+func New(op string, lg *zap.Logger, fields ...Field) *Trace {
+	return &Trace{operation: op, lg: lg, startTime: time.Now(), fields: fields}
+}
+
+// TODO returns a non-nil, empty Trace
+func TODO() *Trace {
+	return &Trace{}
+}
+
+func Get(ctx context.Context) *Trace {
+	if trace, ok := ctx.Value(TraceKey).(*Trace); ok && trace != nil {
+		return trace
+	}
+	return TODO()
+}
+
+func (t *Trace) GetStartTime() time.Time {
+	return t.startTime
+}
+
+func (t *Trace) SetStartTime(time time.Time) {
+	t.startTime = time
+}
+
+func (t *Trace) InsertStep(at int, time time.Time, msg string, fields ...Field) {
+	newStep := step{time, msg, fields}
+	if at < len(t.steps) {
+		t.steps = append(t.steps[:at+1], t.steps[at:]...)
+		t.steps[at] = newStep
+	} else {
+		t.steps = append(t.steps, newStep)
+	}
+}
+
+// Step adds step to trace
+func (t *Trace) Step(msg string, fields ...Field) {
+	if !t.stepDisabled {
+		t.steps = append(t.steps, step{time: time.Now(), msg: msg, fields: fields})
+	}
+}
+
+// DisableStep sets the flag to prevent the trace from adding steps
+func (t *Trace) DisableStep() {
+	t.stepDisabled = true
+}
+
+// EnableStep re-enable the trace to add steps
+func (t *Trace) EnableStep() {
+	t.stepDisabled = false
+}
+
+func (t *Trace) AddField(fields ...Field) {
+	for _, f := range fields {
+		t.fields = append(t.fields, f)
+	}
+}
+
+// Log dumps all steps in the Trace
+func (t *Trace) Log() {
+	t.LogWithStepThreshold(0)
+}
+
+// LogIfLong dumps logs if the duration is longer than threshold
+func (t *Trace) LogIfLong(threshold time.Duration) {
+	if time.Since(t.startTime) > threshold {
+		stepThreshold := threshold / time.Duration(len(t.steps)+1)
+		t.LogWithStepThreshold(stepThreshold)
+	}
+}
+
+// LogWithStepThreshold only dumps step whose duration is longer than step threshold
+func (t *Trace) LogWithStepThreshold(threshold time.Duration) {
+	msg, fs := t.logInfo(threshold)
+	if t.lg != nil {
+		t.lg.Info(msg, fs...)
+	}
+}
+
+func (t *Trace) logInfo(threshold time.Duration) (string, []zap.Field) {
+	endTime := time.Now()
+	totalDuration := endTime.Sub(t.startTime)
+	traceNum := rand.Int31()
+	msg := fmt.Sprintf("trace[%d] %s", traceNum, t.operation)
+
+	var steps []string
+	lastStepTime := t.startTime
+	for _, step := range t.steps {
+		stepDuration := step.time.Sub(lastStepTime)
+		if stepDuration > threshold {
+			steps = append(steps, fmt.Sprintf("trace[%d] '%v' %s (duration: %v)",
+				traceNum, step.msg, writeFields(step.fields), stepDuration))
+		}
+		lastStepTime = step.time
+	}
+
+	fs := []zap.Field{zap.String("detail", writeFields(t.fields)),
+		zap.Duration("duration", totalDuration),
+		zap.Time("start", t.startTime),
+		zap.Time("end", endTime),
+		zap.Strings("steps", steps)}
+	return msg, fs
+}

pkg/traceutil/trace_test.go

@@ -0,0 +1,262 @@
+// Copyright 2019 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package traceutil
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+func TestGet(t *testing.T) {
+	traceForTest := &Trace{operation: "test"}
+	tests := []struct {
+		name        string
+		inputCtx    context.Context
+		outputTrace *Trace
+	}{
+		{
+			name:        "When the context does not have trace",
+			inputCtx:    context.TODO(),
+			outputTrace: TODO(),
+		},
+		{
+			name:        "When the context has trace",
+			inputCtx:    context.WithValue(context.Background(), TraceKey, traceForTest),
+			outputTrace: traceForTest,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			trace := Get(tt.inputCtx)
+			if trace == nil {
+				t.Errorf("Expected %v; Got nil", tt.outputTrace)
+			}
+			if trace.operation != tt.outputTrace.operation {
+				t.Errorf("Expected %v; Got %v", tt.outputTrace, trace)
+			}
+		})
+	}
+}
+
+func TestCreate(t *testing.T) {
+	var (
+		op     = "Test"
+		steps  = []string{"Step1, Step2"}
+		fields = []Field{
+			{"traceKey1", "traceValue1"},
+			{"traceKey2", "traceValue2"},
+		}
+		stepFields = []Field{
+			{"stepKey1", "stepValue2"},
+			{"stepKey2", "stepValue2"},
+		}
+	)
+
+	trace := New(op, nil, fields[0], fields[1])
+	if trace.operation != op {
+		t.Errorf("Expected %v; Got %v", op, trace.operation)
+	}
+	for i, f := range trace.fields {
+		if f.Key != fields[i].Key {
+			t.Errorf("Expected %v; Got %v", fields[i].Key, f.Key)
+		}
+		if f.Value != fields[i].Value {
+			t.Errorf("Expected %v; Got %v", fields[i].Value, f.Value)
+		}
+	}
+
+	for i, v := range steps {
+		trace.Step(v, stepFields[i])
+	}
+
+	for i, v := range trace.steps {
+		if steps[i] != v.msg {
+			t.Errorf("Expected %v; Got %v", steps[i], v.msg)
+		}
+		if stepFields[i].Key != v.fields[0].Key {
+			t.Errorf("Expected %v; Got %v", stepFields[i].Key, v.fields[0].Key)
+		}
+		if stepFields[i].Value != v.fields[0].Value {
+			t.Errorf("Expected %v; Got %v", stepFields[i].Value, v.fields[0].Value)
+		}
+	}
+}
+
+func TestLog(t *testing.T) {
+	tests := []struct {
+		name        string
+		trace       *Trace
+		fields      []Field
+		expectedMsg []string
+	}{
+		{
+			name: "When dump all logs",
+			trace: &Trace{
+				operation: "Test",
+				startTime: time.Now().Add(-100 * time.Millisecond),
+				steps: []step{
+					{time: time.Now().Add(-80 * time.Millisecond), msg: "msg1"},
+					{time: time.Now().Add(-50 * time.Millisecond), msg: "msg2"},
+				},
+			},
+			expectedMsg: []string{
+				"msg1", "msg2",
+			},
+		},
+		{
+			name: "When trace has fields",
+			trace: &Trace{
+				operation: "Test",
+				startTime: time.Now().Add(-100 * time.Millisecond),
+				steps: []step{
+					{
+						time:   time.Now().Add(-80 * time.Millisecond),
+						msg:    "msg1",
+						fields: []Field{{"stepKey1", "stepValue1"}},
+					},
+					{
+						time:   time.Now().Add(-50 * time.Millisecond),
+						msg:    "msg2",
+						fields: []Field{{"stepKey2", "stepValue2"}},
+					},
+				},
+			},
+			fields: []Field{
+				{"traceKey1", "traceValue1"},
+				{"count", 1},
+			},
+			expectedMsg: []string{
+				"Test",
+				"msg1", "msg2",
+				"traceKey1:traceValue1", "count:1",
+				"stepKey1:stepValue1", "stepKey2:stepValue2",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			logPath := filepath.Join(os.TempDir(), fmt.Sprintf("test-log-%d", time.Now().UnixNano()))
+			defer os.RemoveAll(logPath)
+
+			lcfg := zap.NewProductionConfig()
+			lcfg.OutputPaths = []string{logPath}
+			lcfg.ErrorOutputPaths = []string{logPath}
+			lg, _ := lcfg.Build()
+
+			for _, f := range tt.fields {
+				tt.trace.AddField(f)
+			}
+			tt.trace.lg = lg
+			tt.trace.Log()
+			data, err := ioutil.ReadFile(logPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			for _, msg := range tt.expectedMsg {
+				if !bytes.Contains(data, []byte(msg)) {
+					t.Errorf("Expected to find %v in log", msg)
+				}
+			}
+		})
+	}
+}
+
+func TestLogIfLong(t *testing.T) {
+	tests := []struct {
+		name        string
+		threshold   time.Duration
+		trace       *Trace
+		expectedMsg []string
+	}{
+		{
+			name:      "When the duration is smaller than threshold",
+			threshold: time.Duration(200 * time.Millisecond),
+			trace: &Trace{
+				operation: "Test",
+				startTime: time.Now().Add(-100 * time.Millisecond),
+				steps: []step{
+					{time: time.Now().Add(-50 * time.Millisecond), msg: "msg1"},
+					{time: time.Now(), msg: "msg2"},
+				},
+			},
+			expectedMsg: []string{},
+		},
+		{
+			name:      "When the duration is longer than threshold",
+			threshold: time.Duration(50 * time.Millisecond),
+			trace: &Trace{
+				operation: "Test",
+				startTime: time.Now().Add(-100 * time.Millisecond),
+				steps: []step{
+					{time: time.Now().Add(-50 * time.Millisecond), msg: "msg1"},
+					{time: time.Now(), msg: "msg2"},
+				},
+			},
+			expectedMsg: []string{
+				"msg1", "msg2",
+			},
+		},
+		{
+			name:      "When not all steps are longer than step threshold",
+			threshold: time.Duration(50 * time.Millisecond),
+			trace: &Trace{
+				operation: "Test",
+				startTime: time.Now().Add(-100 * time.Millisecond),
+				steps: []step{
+					{time: time.Now(), msg: "msg1"},
+					{time: time.Now(), msg: "msg2"},
+				},
+			},
+			expectedMsg: []string{
+				"msg1",
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			logPath := filepath.Join(os.TempDir(), fmt.Sprintf("test-log-%d", time.Now().UnixNano()))
+			defer os.RemoveAll(logPath)
+
+			lcfg := zap.NewProductionConfig()
+			lcfg.OutputPaths = []string{logPath}
+			lcfg.ErrorOutputPaths = []string{logPath}
+			lg, _ := lcfg.Build()
+
+			tt.trace.lg = lg
+			tt.trace.LogIfLong(tt.threshold)
+			data, err := ioutil.ReadFile(logPath)
+			if err != nil {
+				t.Fatal(err)
+			}
+			for _, msg := range tt.expectedMsg {
+				if !bytes.Contains(data, []byte(msg)) {
+					t.Errorf("Expected to find %v in log", msg)
+				}
+			}
+		})
+	}
+}

raft/confchange/confchange.go

@@ -142,9 +142,6 @@ func (c Changer) Simple(ccs ...pb.ConfChangeSingle) (tracker.Config, tracker.Pro
 	if n := symdiff(incoming(c.Tracker.Voters), incoming(cfg.Voters)); n > 1 {
 		return tracker.Config{}, nil, errors.New("more than one voter changed without entering joint config")
 	}
-	if err := checkInvariants(cfg, prs); err != nil {
-		return tracker.Config{}, tracker.ProgressMap{}, nil
-	}
 
 	return checkAndReturn(cfg, prs)
 }

scripts/build-binary

@@ -57,6 +57,11 @@ function main {
 	cd release
 	setup_env "${PROJ}" "${VER}"
 
+	tarcmd=tar
+	if [[ $(go env GOOS) == "darwin" ]]; then
+		tarcmd=gtar
+	fi
+
 	for os in darwin windows linux; do
 		export GOOS=${os}
 		TARGET_ARCHS=("amd64")
@@ -78,7 +83,7 @@ function main {
 			package "${TARGET}" "${PROJ}"
 
 			if [ ${GOOS} == "linux" ]; then
-				tar cfz "${TARGET}.tar.gz" "${TARGET}"
+				${tarcmd} cfz "${TARGET}.tar.gz" "${TARGET}"
 				echo "Wrote release/${TARGET}.tar.gz"
 			else
 				zip -qr "${TARGET}.zip" "${TARGET}"

scripts/release

@@ -147,7 +147,7 @@ main() {
   # Generate SHA256SUMS
   echo -e "Generating sha256sums of release artifacts.\n"
   pushd ./release
-  grep . -E '\.tar.gz$|\.zip$' | xargs shasum -a 256 > ./SHA256SUMS
+  ls . | grep -E '\.tar.gz$|\.zip$' | xargs shasum -a 256 > ./SHA256SUMS
   popd
   if [ -s ./release/SHA256SUMS ]; then
     cat ./release/SHA256SUMS
@@ -185,7 +185,7 @@ main() {
     docker push "quay.io/coreos/etcd:${RELEASE_VERSION}"
 
     echo "Pushing container images to gcr.io ${RELEASE_VERSION}"
-    gcloud docker -- "push gcr.io/etcd-development/etcd:${RELEASE_VERSION}"
+    gcloud docker -- push "gcr.io/etcd-development/etcd:${RELEASE_VERSION}"
 
     for TARGET_ARCH in "-arm64" "-ppc64le"; do
       echo "Pushing container images to quay.io ${RELEASE_VERSION}${TARGET_ARCH}"

tests/docker-dns-srv/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:17.10
+FROM ubuntu:18.04
 
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections

tests/docker-dns/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:17.10
+FROM ubuntu:18.04
 
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
@@ -20,6 +20,7 @@ RUN apt-get -y update \
   netcat \
   bind9 \
   dnsutils \
+  lsof \
   && apt-get -y update \
   && apt-get -y upgrade \
   && apt-get -y autoremove \

tests/docker-dns/certs-san-dns/Procfile

@@ -0,0 +1,6 @@
+# Use goreman to run `go get github.com/mattn/goreman`
+etcd1: ./etcd --name m1 --data-dir /tmp/m1.data --listen-client-urls https://127.0.0.1:2379 --advertise-client-urls https://m1.etcd.local:2379 --listen-peer-urls https://127.0.0.1:2380 --initial-advertise-peer-urls=https://m1.etcd.local:2380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-san-dns/server-1.crt --peer-key-file=/certs-san-dns/server-1.key.insecure --peer-trusted-ca-file=/certs-san-dns/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn etcd.local --cert-file=/certs-san-dns/server-1.crt --key-file=/certs-san-dns/server-1.key.insecure --trusted-ca-file=/certs-san-dns/ca.crt --client-cert-auth --logger=zap --log-outputs=stderr
+
+etcd2: ./etcd --name m2 --data-dir /tmp/m2.data --listen-client-urls https://127.0.0.1:22379 --advertise-client-urls https://m2.etcd.local:22379 --listen-peer-urls https://127.0.0.1:22380 --initial-advertise-peer-urls=https://m2.etcd.local:22380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-san-dns/server-2.crt --peer-key-file=/certs-san-dns/server-2.key.insecure --peer-trusted-ca-file=/certs-san-dns/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn etcd.local --cert-file=/certs-san-dns/server-2.crt --key-file=/certs-san-dns/server-2.key.insecure --trusted-ca-file=/certs-san-dns/ca.crt --client-cert-auth --logger=zap --log-outputs=stderr
+
+etcd3: ./etcd --name m3 --data-dir /tmp/m3.data --listen-client-urls https://127.0.0.1:32379 --advertise-client-urls https://m3.etcd.local:32379 --listen-peer-urls https://127.0.0.1:32380 --initial-advertise-peer-urls=https://m3.etcd.local:32380 --initial-cluster-token tkn --initial-cluster=m1=https://m1.etcd.local:2380,m2=https://m2.etcd.local:22380,m3=https://m3.etcd.local:32380 --initial-cluster-state new --peer-cert-file=/certs-san-dns/server-3.crt --peer-key-file=/certs-san-dns/server-3.key.insecure --peer-trusted-ca-file=/certs-san-dns/ca.crt --peer-client-cert-auth --peer-cert-allowed-cn etcd.local --cert-file=/certs-san-dns/server-3.crt --key-file=/certs-san-dns/server-3.key.insecure --trusted-ca-file=/certs-san-dns/ca.crt --client-cert-auth --logger=zap --log-outputs=stderr

tests/docker-dns/certs-san-dns/ca-csr.json

@@ -0,0 +1,19 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "ca",
+  "ca": {
+    "expiry": "87600h"
+  }
+}

tests/docker-dns/certs-san-dns/ca.crt

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDrjCCApagAwIBAgIUV77P/m6U+QIMz7Ql0Q6xC3GO/fAwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDcyMjIyMDBaFw0yOTEwMDQyMjIy
+MDBaMG8xDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTELMAkGA1UEAxMCY2EwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK
+AoIBAQDEN9lZnkS16bi42zl+iGlYSHGJn0uxiqhff1KRJlwbEBXr3ywllJLgnAA3
+XEQsBMYk0yEB82380flVJd6UMt+0n6bo5Mp2Z+X8eXZgVgB4uLz0APRhozO89I2D
+wk74aTrV3wseCmN9ZOvG+2b1AzM6rwwnozhnoC2qlZ5yNZRSKMTRX+ZcDQ6FQopk
+Kg+ACGyiU94bLJkd4Vj7oSOiParjtj1laGE88QAL8clkcT6enHlwVJDs7BF3SRBI
+sBKlUnyC47mjR4v9KKkeZ7LHBcW9D7FZZYNg85mubVHfj8rZb1EAF+Kqskd6YpYz
+ZezQVdJOyUrp8/+mSBaS2HpF4HjpAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAP
+BgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTr390x+ChxCV+AkCnxh+5vgtoiyTAN
+BgkqhkiG9w0BAQsFAAOCAQEAq+o4uF9xkJ/SzGgBePb3r/F0aNcBIY3XmCsGE4gd
+0U/tqkGP10BKlermi87ADLxjBux+2n6eAHycac9mDynOr1d5GUVHK8BrAzKeabuP
+Q8J2NQyVXpRF9z2EolLpw7J1n5CYJqsVMBjov33AKk9SmCFg3O4wD6oladWXT/Ie
+ld2+EUS6TLzPNsU+AoPx64L0Aru05ynpPnlUB+DSXCBUckffmGgv0HEd5bU3QOl4
+9SUx35lk8nh7x+sHQblijuNNLi7bTIhzQTolJTCo3rd8YgSdnof0z5bROVTwymD5
+tWshIE4BP+ri+1NPKCe2KlcP3MIynKtx+obr5cLZjDHWoA==
+-----END CERTIFICATE-----

tests/docker-dns/certs-san-dns/gencert.json

@@ -0,0 +1,13 @@
+{
+  "signing": {
+    "default": {
+        "usages": [
+          "signing",
+          "key encipherment",
+          "server auth",
+          "client auth"
+        ],
+        "expiry": "87600h"
+    }
+  }
+}

tests/docker-dns/certs-san-dns/gencerts.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+if ! [[ "$0" =~ "./gencerts.sh" ]]; then
+	echo "must be run from 'fixtures'"
+	exit 255
+fi
+
+if ! which cfssl; then
+	echo "cfssl is not installed"
+	exit 255
+fi
+
+cfssl gencert --initca=true ./ca-csr.json | cfssljson --bare ./ca
+mv ca.pem ca.crt
+openssl x509 -in ca.crt -noout -text
+
+# generate wildcard certificates DNS: m1/m2/m3.etcd.local
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr-1.json | cfssljson --bare ./server-1
+mv server-1.pem server-1.crt
+mv server-1-key.pem server-1.key.insecure
+
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr-2.json | cfssljson --bare ./server-2
+mv server-2.pem server-2.crt
+mv server-2-key.pem server-2.key.insecure
+
+cfssl gencert \
+    --ca ./ca.crt \
+    --ca-key ./ca-key.pem \
+    --config ./gencert.json \
+    ./server-ca-csr-3.json | cfssljson --bare ./server-3
+mv server-3.pem server-3.crt
+mv server-3-key.pem server-3.key.insecure
+
+rm -f *.csr *.pem *.stderr *.txt

tests/docker-dns/certs-san-dns/run.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+rm -rf /tmp/m1.data /tmp/m2.data /tmp/m3.data
+
+/etc/init.d/bind9 start
+
+# get rid of hosts so go lookup won't resolve 127.0.0.1 to localhost
+cat /dev/null >/etc/hosts
+echo "127.0.0.1 m1.etcd.local" >> /etc/hosts
+echo "127.0.0.1 m2.etcd.local" >> /etc/hosts
+echo "127.0.0.1 m3.etcd.local" >> /etc/hosts
+
+goreman -f /certs-san-dns/Procfile start &
+# TODO: remove random sleeps
+sleep 7s
+
+ETCDCTL_API=3 ./etcdctl \
+ --cacert=/certs-san-dns/ca.crt \
+ --cert=/certs-san-dns/server-1.crt \
+ --key=/certs-san-dns/server-1.key.insecure \
+ --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
+ endpoint health --cluster
+
+printf "\nPut abc \n"
+ETCDCTL_API=3 ./etcdctl \
+ --cacert=/certs-san-dns/ca.crt \
+ --cert=/certs-san-dns/server-2.crt \
+ --key=/certs-san-dns/server-2.key.insecure \
+ --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
+ put abc def
+
+printf "\nGet abc \n"
+ETCDCTL_API=3 ./etcdctl \
+ --cacert=/certs-san-dns/ca.crt \
+ --cert=/certs-san-dns/server-3.crt \
+ --key=/certs-san-dns/server-3.key.insecure \
+ --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
+ get abc
+
+printf "\nKill etcd server 1...\n"
+kill $(lsof -t -i:2379)
+sleep 7s
+
+printf "\nGet abc after killing server 1\n"
+ETCDCTL_API=3 ./etcdctl \
+ --cacert=/certs-san-dns/ca.crt \
+ --cert=/certs-san-dns/server-2.crt \
+ --key=/certs-san-dns/server-2.key.insecure \
+ --endpoints=https://m1.etcd.local:2379,https://m2.etcd.local:22379,https://m3.etcd.local:32379 \
+ get abc
+printf "\n\nDone!!!\n\n"
+

tests/docker-dns/certs-san-dns/server-1.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEDzCCAvegAwIBAgIUYSODFGYUNAEskvyamAAxpZ8/86swDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDcyMjIyMDBaFw0yOTEwMDQyMjIy
+MDBaMHcxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTETMBEGA1UEAxMKZXRjZC5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBAMNEq66ZcntXibYne3W9L53JyMmGrJJi2FbVAEv76OraVnO5
+7qJNXjXZ3bOhQ3WDawbWBA5lNi1mwZcKVxM41PQXpez/6/ZkZliwNQFsDZ3WgPIx
+mfcWWnoVPEKFrJTnKZm5/o+50w07yMGZLCgIS66oIcOGJ3G35/NKm+T94yKnRV2m
+M1YvkmgU69MwQwbvGh1fypKB734wVp9Yz46FTuAoY8I63feYrSHKHXZf70rm3Kqm
+iTU3jixWq86aI1dIRbAqObc5pgSoBwAczLjWvhhcO7n9KRkyzxjg+ZFPwRHiBWi1
+ZU70D4XHZMdcAgu+2/IBXfGBZbKOyq9WN65N9tUCAwEAAaOBmjCBlzAOBgNVHQ8B
+Af8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB
+/wQCMAAwHQYDVR0OBBYEFJzBC8YE22RmOwKyxnH0SPC08zE4MB8GA1UdIwQYMBaA
+FOvf3TH4KHEJX4CQKfGH7m+C2iLJMBgGA1UdEQQRMA+CDW0xLmV0Y2QubG9jYWww
+DQYJKoZIhvcNAQELBQADggEBAKvIARZDTNcGAcu5SkrjB/mWlq7GaLqgnGARvMQ0
+O5IC6hPsOcIsTnGKzert2xkc6y7msYMOl4ddP5PgSIfpCtkmL6bACoros4ViWwl5
+Lg0YF3PQvwSL+h2StTE2pGrNp/eQL8HJD2Lhyac2vTAq01Vbh3ySrfQP9zjoH8U7
++mJJk9VWAagU+ww17kq5VZL9iJnlFSxVLNo6dcNo/dU6eWqKWoZjAHl+/zhoSOuZ
+tBRshTcFuLbBe59ULFoZ+Mt5Sa4+OuN5Jir4hQH6DS1ETd7hwsSvHf6KcIw9fIXz
+h+PZ0ssNDq4Yr7i3dQS5xAQO1aO35Ru9q2ABt20E1dQGIyY=
+-----END CERTIFICATE-----

tests/docker-dns/certs-san-dns/server-1.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpQIBAAKCAQEAw0Srrplye1eJtid7db0vncnIyYaskmLYVtUAS/vo6tpWc7nu
+ok1eNdnds6FDdYNrBtYEDmU2LWbBlwpXEzjU9Bel7P/r9mRmWLA1AWwNndaA8jGZ
+9xZaehU8QoWslOcpmbn+j7nTDTvIwZksKAhLrqghw4Yncbfn80qb5P3jIqdFXaYz
+Vi+SaBTr0zBDBu8aHV/KkoHvfjBWn1jPjoVO4Chjwjrd95itIcoddl/vSubcqqaJ
+NTeOLFarzpojV0hFsCo5tzmmBKgHABzMuNa+GFw7uf0pGTLPGOD5kU/BEeIFaLVl
+TvQPhcdkx1wCC77b8gFd8YFlso7Kr1Y3rk321QIDAQABAoIBAQCl3c4LqNKDDQ+w
+SAdqMsKgwIerD5fFXOsxjwsKgDgQTljDQrv+58NP8PmOnTxFNNWT3/VgGP8VP8TP
+vPvMGylhEjligN151TzOtxa/V36VhWDQ2etT5IwEScd/Jjc74MQIjeI7SfiJtC/K
+q4bDlpBbEvxjLrCQu0vu8IBN2o+2nWx8l7Jy0VrDuw5LQM90ZA7OcU7H2kE1ehbp
+M5waHE0tdgHzlLqrVl0RlXh/FlIG7/cfQRL1rpD5T8llD7XshF2BhtXerk+QtC9b
+It8xGnhd6e9Yk96KIN/3U/W5DORYwtq1r54r1OxZkUX3C0RqU2P3EcNvBHbbZydm
+6xq6EfDBAoGBAM3LIHo4v96YPNHtj+GI9ZRr+o9UMrl3dcRTMkbEjFIqBdON8GrS
+fdLSvZms+wqU8v7gNEZhhk7U9Y71pHUZsN5WAGHMCC6Q6/5lY2ObEEitrV7btrUe
+75JNlSq52JT7L9NZRhD5ACqw9qrdUq0mNyPtrSV/J2DfubuBWcSLf58lAoGBAPLo
+MGLyzuG5WTwPAkcB/T3Z5kNFlr8po9tuso5WDuXws7nIPR8yb6UIvP7JqWOgaHyh
+YBA4aKC1T8gpAwVxZxJ9bbntxt13sxyuMZgA/CGn6FXCPbhAztnQDle81QcsMGXK
+y2YbeMUVuMrowcjK6g8J9E9AkB4SDvme+xhEQgHxAoGBAIxtzRa5/Ov3dKFH+8PK
+QtJqMIt3yDlZNEqo/wjdfGdg96LaG7G5O1UOq4TfTlt1MrAL7IAOcqj+lyZbp0Kl
+KlU92Hrj0L199RwesYi5uo3tvf2Z7n5/wrlSKbUDJrDbC1Kse6x/TcbUBS6pYo53
+Im9o85s/vm5TnJk/9jKxgn/lAoGAVUbutc5IkzZe/ZbHVeZ84Zn+HN/xbGtR+1eB
+mDbeRBuc/TwvOSSbzXSj5U8nCLLn+9krwIYNNV5yA/Nh/Ccz6Gnge8XeayH637bH
+8nVmDurDxlfLE0StWgqQ/nxszXfWBeaMQeyjGY3mslXEspmKUn1MKAaikewFFd2a
+iYptIgECgYEAr81jSoXyHSKpEEHzy5hyH+JOsUeWZVFduqkTTHNZe7MlXSSSZdhW
+6TCjnA9HpzBlgTI8PwXXKEa2G7OCr4dHFBJSWCgzQTfd1hf5xiE7ca2bxiEC7SKF
+H3TvfLCi9Dky9uFAXsp6SlI/x6Abm6CpqTlR19KyCo64LztaAmRkmNU=
+-----END RSA PRIVATE KEY-----

tests/docker-dns/certs-san-dns/server-2.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEDzCCAvegAwIBAgIUDrW+8pB5rh4jfT8GQ3R9EqRLuzkwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDcyMjIyMDBaFw0yOTEwMDQyMjIy
+MDBaMHcxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTETMBEGA1UEAxMKZXRjZC5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBANHtpC3LDlC1MqHx/LT6vWA9DJApziy9Vh0f0SC1hFjRiFGp
+yA8d4uWHg7ebEVj/hWyJPrYpNMSDXhmJVa8UtE6G3B2ZS4WZsjfKMYs0ydu8mjjV
+FlfC6vuDGX3gUdI7XhW1KCmnFI0XfRaskS/khY31SMyblAZ0hDpRz/nQ3vyMSS7+
+xYgPn7SHNrJFz8+K3NB35lbvkBvYZvVJ0mONeIMB1BffHILzexiaXyHXeKTPw9yI
+FSRTDlXQqY9afNpAAv12xW2Xa9chuQ5Q+5P8syRqePgjR+TVJkeUCpLunNHcxZTD
+DoXqJjOlqy6OzdFGnGzvtDh/1/QL880/e6jOCcUCAwEAAaOBmjCBlzAOBgNVHQ8B
+Af8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB
+/wQCMAAwHQYDVR0OBBYEFNoiUFY7gFUJUtJpBXFVIFipiFo/MB8GA1UdIwQYMBaA
+FOvf3TH4KHEJX4CQKfGH7m+C2iLJMBgGA1UdEQQRMA+CDW0yLmV0Y2QubG9jYWww
+DQYJKoZIhvcNAQELBQADggEBAGUisaOqg4ps1XTqRnqnk/zajC0MeyayE4X2VLtO
+gq04wT7N9nUmFAiL2uUBzK4IKrb68ZIGQbm/3NNJrKxauWvK79J9xbeOAnOQWIAx
+VFA7uGw0JpiYFk6W9YzTCpNlIWEOEw5RaNIj8F5dAFqgqNDqd1zw1+04jIGlBTpD
+v3LQjr8IvB/cmvnugwAnb8cKDlr1GO322/1otrJi2BpmjAi4FQmuxdyQTmgkQU7T
+k2whauuwDrwVmc+LyoObbiiaJPi60lSABIttbUmFqWo9U+mBcbAtFE6EW6Wo1gFR
+q7uKqwYjARW/h/amHhyiHkNnu+TjY1SL2+kk+EBAt0SSmq8=
+-----END CERTIFICATE-----

tests/docker-dns/certs-san-dns/server-2.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA0e2kLcsOULUyofH8tPq9YD0MkCnOLL1WHR/RILWEWNGIUanI
+Dx3i5YeDt5sRWP+FbIk+tik0xINeGYlVrxS0TobcHZlLhZmyN8oxizTJ27yaONUW
+V8Lq+4MZfeBR0jteFbUoKacUjRd9FqyRL+SFjfVIzJuUBnSEOlHP+dDe/IxJLv7F
+iA+ftIc2skXPz4rc0HfmVu+QG9hm9UnSY414gwHUF98cgvN7GJpfIdd4pM/D3IgV
+JFMOVdCpj1p82kAC/XbFbZdr1yG5DlD7k/yzJGp4+CNH5NUmR5QKku6c0dzFlMMO
+heomM6WrLo7N0UacbO+0OH/X9AvzzT97qM4JxQIDAQABAoIBAQCYEZ9mlLsv97JP
+4a1/pErelhqtq7rwac8hldS17weKF266SVTkrm+YeYwOysPMRnzuXJUS+9J/r/HQ
+ac2p4EOkxshYoJ02kFmrVEqDXqADDyJgnOtsc4Qo4ZTrvD1JHzxOWUZYtfGLK0Kv
+1B3wJYghh1dO8DxQWMMYQ/92JboCEoVmO/vAcUH5V4qhZMEGvFm8AiaXnVi13myt
+OAlfyQQ1CsnOoxxQhomzqNVrMjPelv5yLAq1Z5gXSeylc6y8NVWKsLbWJUj5IhqH
+bmCw2V/1snJCJews/S/4wgDBibjldlUEPfjNwBoeRTl9DB6uCHzUiF98PB8MoDx5
+VaJiRHZZAoGBAOqVcgB+3gJ9Pf+6bUdL4NhKdr4wje2IAbeidQMXOsbp455b7NLj
+/Z92tKOGJ2HBdGBzGkA4JbHcy/HBxDm6DXKWIIqYcOubDDDiBAYtEJhLG3Mqz4p8
+sp1QUICQoskCAP4gHc8/AeXKp1CQoU1dJksC4mZ66KQMdYaJ1f7gNxJ7AoGBAOUX
+9mLDFjqpJ7IPt02I4yn/tlFI3GLwuO/yxEuCGt8T2CAXkc/cp+ojEI29ckwYpqv6
+D+FRPYqNN+c6OJWAR4U4OiuRQlShGZmBvn11BIn7ILZ3KnxvFXKkOzzFNU5oYczE
+/L/z2SSKQfGlgDWmKWIoWt5D3TjMA7xysTgQIcC/AoGAFgyV+pXyKCm9ehv7yYfI
+Sow1PQszS/BMuQX8GZ5FWA0D6A6b4/aqECMIN5aUfQvB9I7dGMwuPtmSEdc0qnhi
+azLRPDW3521bZ/zWg/4YYTguDFUpzMqLv12dM3hk1J/rl/dM1f4GH6M8tsXhY3Qt
+9T8AKMHEvCavpUWvZ5WLl6ECgYAgxmzZdE+Z1Nl5AAaZcRwOxiavOl1NSmMq8PBk
+XRi7EXu6G6Ugt9DODnYv0QqpGF2//OaItba4O7vjuNCfktqolIK9+OokcWfYLley
+WytrEiJ7+FB7vOi0ngpbh1s4/HYBda0zSQ+nyp/kkmjlRABnqp5VbiAYIBfovf/c
+pXIuwQKBgQCGJBX7vmFcsL1qdG5d8jQr2K/dbTcU8sXQzUIXGQcCxePYOrO8Rcn2
+EMXAGIdOn6i2x0/rNn+EnPHhT6XC0hSOu52srL8BB9tbDYk3i+3ghUG5QI4dp+GQ
+D1+HZD3SVrqjWlTU0aBB/NYMldIo9e3LU1ZUXTm2Rmg6Mre9ann6/w==
+-----END RSA PRIVATE KEY-----

tests/docker-dns/certs-san-dns/server-3.crt

@@ -0,0 +1,24 @@
+-----BEGIN CERTIFICATE-----
+MIIEDzCCAvegAwIBAgIUNPjXxMAkrxdr1sZA7Gw+gYbVeLAwDQYJKoZIhvcNAQEL
+BQAwbzEMMAoGA1UEBhMDVVNBMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMQ0wCwYDVQQKEwRldGNkMRYwFAYDVQQLEw1ldGNkIFNl
+Y3VyaXR5MQswCQYDVQQDEwJjYTAeFw0xOTEwMDcyMjIyMDBaFw0yOTEwMDQyMjIy
+MDBaMHcxDDAKBgNVBAYTA1VTQTETMBEGA1UECBMKQ2FsaWZvcm5pYTEWMBQGA1UE
+BxMNU2FuIEZyYW5jaXNjbzENMAsGA1UEChMEZXRjZDEWMBQGA1UECxMNZXRjZCBT
+ZWN1cml0eTETMBEGA1UEAxMKZXRjZC5sb2NhbDCCASIwDQYJKoZIhvcNAQEBBQAD
+ggEPADCCAQoCggEBALwQOtWoCcO13D/7i96Bkb376WvoqYJw+yN9kYwVkpM1+EQd
+3hzSNT0byRGeNtlXAd8tY/SpjTM7mnq5yIhNjhJ2eo5GO1YuJyDJe9WnfQ30rVfv
+WzCV/BiwloaqX/tlgCJ3PVNAZdyCZ+ouRIggBUHCQo88LuKwpM9QrUmBCGFLD/M2
+PYKewGv+h9JwMRLxp5mARBS+bkUsQy9F7U/GZs/9xULXIo9l3Bj8Zqz6UMmtW+Y2
+lkK5wawG04bZwkr8lUzMC2AVKFidTuZsda9GP4OxKclW0ro0HtlYaiI7+a0xONZ6
+yuj4cYrs1KZ9z3uYji1Li8XFUb4g/v9dar0oK70CAwEAAaOBmjCBlzAOBgNVHQ8B
+Af8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB
+/wQCMAAwHQYDVR0OBBYEFATpeRk6Bxgf8LHU/wlw0iLQltEoMB8GA1UdIwQYMBaA
+FOvf3TH4KHEJX4CQKfGH7m+C2iLJMBgGA1UdEQQRMA+CDW0zLmV0Y2QubG9jYWww
+DQYJKoZIhvcNAQELBQADggEBADjH3ytTogX2BqnhYaVia31Zjy240iViU6BNCARq
+PdBB5WCtti7yzonfS9Uytc9YLB4ln4Z0wZpRk3O0QGehHX5CDT5EL5zKwDQdoYG3
+oKx9qOu2VyxDA/1hYdPvMW3aq4g/oE8nFjNbrFEVCuGLbJdfDnyJJFsvNRNqs8hS
+xpfYLNH9lD4sD13vul7RJQJrvCjbaqQp9oLe9NZ9f+cBPGqATkicMWbABq4xbpCE
+IY19SHk0WHRSem5jlbfF3O58Ow+LRR/Bn2/IYKpyidEixxu9VX06BDRH5GmG7wBd
+5Y9YhmeyPCXiHHPar7m/Rmel82RLI+/qomKh9pii3u357yY=
+-----END CERTIFICATE-----

tests/docker-dns/certs-san-dns/server-3.key.insecure

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAvBA61agJw7XcP/uL3oGRvfvpa+ipgnD7I32RjBWSkzX4RB3e
+HNI1PRvJEZ422VcB3y1j9KmNMzuaernIiE2OEnZ6jkY7Vi4nIMl71ad9DfStV+9b
+MJX8GLCWhqpf+2WAInc9U0Bl3IJn6i5EiCAFQcJCjzwu4rCkz1CtSYEIYUsP8zY9
+gp7Aa/6H0nAxEvGnmYBEFL5uRSxDL0XtT8Zmz/3FQtcij2XcGPxmrPpQya1b5jaW
+QrnBrAbThtnCSvyVTMwLYBUoWJ1O5mx1r0Y/g7EpyVbSujQe2VhqIjv5rTE41nrK
+6PhxiuzUpn3Pe5iOLUuLxcVRviD+/11qvSgrvQIDAQABAoIBAG1ny7JsFXIjpEQc
+pJwHKLArkvnR2nsmGxPkgv3JtwGpDgsijQqbR5mLRofXUPVTZqVdFJ9K2/gIHrBy
+0DRrWdFn15hZRz+1jdHHJSGAVIH/67AScSxstMHwSUGCcGAiBk8Gq0h5WEjWHHnh
+/MBsUGKXDn2hd20tclOhDY6LYEKolRPFjfBmPRdhdR5A6RS+U+jx1yFsWa6cUjv6
+kInlE5yMdhEOuA/QnVvcaAsKb5CKAuCtAkmFH3fjDp3nkhYFXJy4DTsVRMAfsr5s
+SpsKt272URd5fLeZ5QlOb82QCvJr9GushkkKk7N5TMh5C/r74zpROdLTRlXD4I2q
+yvnSv8kCgYEA+HRjeRRxujVWo7YSnHYJ/xConrCSekfRMvIXvSq43E+I/t5SlPl8
+YoJYhGWzZ7A/szqTvTW/v2blScd+X4KiK0TX8tTQFvWEBBcZhLILUB/ZiIfi/6ZG
+fxe+BAmTMSBThknnRsvAA4jkTvErdpBhhRltyjdLunEEjnfSzJJORHMCgYEAwcZU
+TpAfo4ni1Am9Nskk/5LjmPX5u+qfPNJfe6dfO+BoMA51XuAagqZhdsSwTGoxs5xQ
+cKmNFA6QmAQnPZK7+QYwmDUXb8/Dtz/d5jylsZdYRHYr4hx3DcKFFEyhlPqrj44k
+HxparrkDIq7nVz1t3YMVXYJM/5k2cx/VHlTD8w8CgYEA6Ypl0nNwL4thpENKHT4r
+SVG8XmY1WbHWKCA+Rjc5SwWMDZ6nW5dj3ykM0W7Tg5y9U9i09L7oPZ8X2hEmbdra
+Wve8UWrPKzWe4UVhXEULs0Ys8VRiANKoI2EK4LqrXBs5x9oCBp8RH4F2semqZCl1
+MWpktBbkHR2NHenuARNpdJcCgYBzlY3sXuPAdRssR7Lp3wmGuWOxdefFQ6pAaWwz
+Ih8YZD9Bix5PvXWSwRQZ+DEBI8cJ0A/bZAeXEykExFVz0Pb3D84kvGaCd3fS8vG1
+yC89w30POT3r3fbV6lXfSeaIKw3yz2KUeu/kkM9h/NpZm3bRTsOLx5GOVSG5gh9p
+vD412QKBgFxq4rsxJC6+QZvRZaJDcmTHSytbAw3B5Lyv6G+xLBUqc27KjQzCved1
+9Ofzy7KEC3AtKiq3Y0q5q01Rzk5ZYCh6lVe2tw36Muw1bvZjqblGm9X2VRO8Ui2Q
+4WOdvIP4z5ZTJQXdIahKAYOyxiYFIvCkvS5SYoKkgWNSzFNKvQtH
+-----END RSA PRIVATE KEY-----

tests/docker-dns/certs-san-dns/server-ca-csr-1.json

@@ -0,0 +1,19 @@
+{
+  "key": {
+    "algo": "rsa",
+    "size": 2048
+  },
+  "names": [
+    {
+      "O": "etcd",
+      "OU": "etcd Security",
+      "L": "San Francisco",
+      "ST": "California",
+      "C": "USA"
+    }
+  ],
+  "CN": "etcd.local",
+  "hosts": [
+    "m1.etcd.local"
+  ]
+}

tests/docker-dns/certs-san-dns/server-ca-csr-2.json

@@ -0,0 +1,19 @@
+{
+    "key": {
+      "algo": "rsa",
+      "size": 2048
+    },
+    "names": [
+      {
+        "O": "etcd",
+        "OU": "etcd Security",
+        "L": "San Francisco",
+        "ST": "California",
+        "C": "USA"
+      }
+    ],
+    "CN": "etcd.local",
+    "hosts": [
+      "m2.etcd.local"
+    ]
+  }

tests/docker-dns/certs-san-dns/server-ca-csr-3.json

@@ -0,0 +1,19 @@
+{
+    "key": {
+      "algo": "rsa",
+      "size": 2048
+    },
+    "names": [
+      {
+        "O": "etcd",
+        "OU": "etcd Security",
+        "L": "San Francisco",
+        "ST": "California",
+        "C": "USA"
+      }
+    ],
+    "CN": "etcd.local",
+    "hosts": [
+      "m3.etcd.local"
+    ]
+  }

tests/docker-static-ip/Dockerfile

@@ -1,4 +1,4 @@
-FROM ubuntu:17.10
+FROM ubuntu:18.04
 
 RUN rm /bin/sh && ln -s /bin/bash /bin/sh
 RUN echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections