Merge pull request #9323 from gyuho/functional-tester-guide

*: clean up functional tester guide with local cluster, docker container

Dockerfile-functional-tester

@@ -47,7 +47,7 @@ RUN go get -v github.com/coreos/gofail \
   && cp ./bin/etcd-agent /etcd-agent \
   && cp ./bin/etcd-tester /etcd-tester \
   && cp ./bin/etcd-runner /etcd-runner \
-  && go build -v -o /benchmark ./cmd/tools/benchmark \
-  && go build -v -o /etcd-test-proxy ./cmd/tools/etcd-test-proxy \
+  && go build -v -o /benchmark ./tools/benchmark \
+  && go build -v -o /etcd-test-proxy ./tools/etcd-test-proxy \
   && popd \
   && rm -rf ${GOPATH}/src/github.com/coreos/etcd

hack/scripts-dev/Makefile

@@ -418,7 +418,7 @@ docker-dns-srv-test-certs-wildcard-run:
 #   make build-etcd-test-proxy -f ./hack/scripts-dev/Makefile
 
 build-etcd-test-proxy:
-	go build -v -o ./bin/etcd-test-proxy ./cmd/tools/etcd-test-proxy
+	go build -v -o ./bin/etcd-test-proxy ./tools/etcd-test-proxy
 
 
 

tools/etcd-test-proxy/README.md

@@ -20,7 +20,7 @@ $ ETCDCTL_API=3 ./bin/etcdctl --endpoints localhost:23790 put foo bar
 Proxy overhead per request is under 500μs
 
 ```bash
-$ go build -v -o ./bin/benchmark ./cmd/tools/benchmark
+$ go build -v -o ./bin/benchmark ./tools/benchmark
 
 $ ./bin/benchmark \
   --endpoints localhost:2379 \

tools/functional-tester/README.md

@@ -4,7 +4,7 @@ etcd functional test suite tests the functionality of an etcd cluster with a foc
 
 etcd functional test suite has two components: etcd-agent and etcd-tester. etcd-agent runs on every test machines and etcd-tester is a single controller of the test. etcd-tester controls all the etcd-agent to start etcd clusters and simulate various failure cases.
 
-## requirements
+## Requirements
 
 The environment of the cluster must be stable enough, so etcd test suite can assume that most of the failures are generated by itself.
 
@@ -18,8 +18,8 @@ etcd functional tester control the progress of the functional tests. It calls th
 
 ### Run locally
 
-```
-$ PASSES=functional ./test
+```bash
+PASSES=functional ./test
 ```
 
 ### Run with Docker
@@ -29,8 +29,7 @@ To run locally, first build tester image:
 ```bash
 pushd ../..
 
-GO_VERSION=1.9.3 \
-  make build-docker-functional-tester \
+make build-docker-functional-tester \
   -f ./hack/scripts-dev/Makefile
 
 popd
@@ -39,9 +38,20 @@ popd
 And run [example scripts](./scripts).
 
 ```bash
-./scripts/agent-1.sh
-./scripts/agent-2.sh
-./scripts/agent-3.sh
-
-./scripts/tester-limit.sh
+# run 3 agents for 3-node local etcd cluster
+./scripts/docker-local-agent.sh 1
+./scripts/docker-local-agent.sh 2
+./scripts/docker-local-agent.sh 3
+
+# to run only 1 tester round
+LIMIT=1 ./scripts/docker-local-tester.sh
+
+# to run long-running tests with no limit
+LIMIT=1 ./scripts/docker-local-tester.sh
+
+# to run only 1 tester round with election runner and others
+# default is STRESSER="keys,lease"
+LIMIT=1 \
+  STRESSER="keys,lease,election-runner,watch-runner,lock-racer-runner,lease-runner" \
+  ./scripts/docker-local-tester.sh
 ```

tools/functional-tester/scripts/agent-1.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-<<COMMENT
-# to run agent
-./scripts/agent-1.sh
-
-# to run with failpoints
-ETCD_EXEC_PATH=/etcd-failpoints ./scripts/agent-1.sh
-COMMENT
-
-if ! [[ "$0" =~ "scripts/agent-1.sh" ]]; then
-  echo "must be run from tools/functional-tester"
-  exit 255
-fi
-
-if [ -z "${ETCD_EXEC_PATH}" ]; then
-  ETCD_EXEC_PATH=/etcd
-	echo "Running agent without failpoints:" ${ETCD_EXEC_PATH}
-elif [[ "${ETCD_EXEC_PATH}" == "/etcd-failpoints" ]]; then
-	echo "Running agent with failpoints:" ${ETCD_EXEC_PATH}
-else
-  echo "Cannot find executable:" ${ETCD_EXEC_PATH}
-  exit 255
-fi
-
-rm -rf `pwd`/agent-1 && mkdir -p `pwd`/agent-1
-docker run \
-  --rm \
-  --net=host \
-  --name agent-1 \
-  --mount type=bind,source=`pwd`/agent-1,destination=/agent-1 \
-  gcr.io/etcd-development/etcd-functional-tester:go1.9.3 \
-  /bin/bash -c "/etcd-agent \
-    --etcd-path ${ETCD_EXEC_PATH} \
-    --etcd-log-dir /agent-1 \
-    --port :19027 \
-    --failpoint-addr :7381"

tools/functional-tester/scripts/agent-2.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-<<COMMENT
-# to run agent
-./scripts/agent-2.sh
-
-# to run with failpoints
-ETCD_EXEC_PATH=/etcd-failpoints ./scripts/agent-2.sh
-COMMENT
-
-if ! [[ "$0" =~ "scripts/agent-2.sh" ]]; then
-  echo "must be run from tools/functional-tester"
-  exit 255
-fi
-
-if [ -z "${ETCD_EXEC_PATH}" ]; then
-  ETCD_EXEC_PATH=/etcd
-	echo "Running agent without failpoints:" ${ETCD_EXEC_PATH}
-elif [[ "${ETCD_EXEC_PATH}" == "/etcd-failpoints" ]]; then
-	echo "Running agent with failpoints:" ${ETCD_EXEC_PATH}
-else
-  echo "Cannot find executable:" ${ETCD_EXEC_PATH}
-  exit 255
-fi
-
-rm -rf `pwd`/agent-2 && mkdir -p `pwd`/agent-2
-docker run \
-  --rm \
-  --net=host \
-  --name agent-2 \
-  --mount type=bind,source=`pwd`/agent-2,destination=/agent-2 \
-  gcr.io/etcd-development/etcd-functional-tester:go1.9.3 \
-  /bin/bash -c "/etcd-agent \
-    --etcd-path ${ETCD_EXEC_PATH} \
-    --etcd-log-dir /agent-2 \
-    --port :29027 \
-    --failpoint-addr :7382"

tools/functional-tester/scripts/agent-3.sh

@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-
-<<COMMENT
-# to run agent
-./scripts/agent-3.sh
-
-# to run with failpoints
-ETCD_EXEC_PATH=/etcd-failpoints ./scripts/agent-3.sh
-COMMENT
-
-if ! [[ "$0" =~ "scripts/agent-3.sh" ]]; then
-  echo "must be run from tools/functional-tester"
-  exit 255
-fi
-
-if [ -z "${ETCD_EXEC_PATH}" ]; then
-  ETCD_EXEC_PATH=/etcd
-	echo "Running agent without failpoints:" ${ETCD_EXEC_PATH}
-elif [[ "${ETCD_EXEC_PATH}" == "/etcd-failpoints" ]]; then
-	echo "Running agent with failpoints:" ${ETCD_EXEC_PATH}
-else
-  echo "Cannot find executable:" ${ETCD_EXEC_PATH}
-  exit 255
-fi
-
-rm -rf `pwd`/agent-3 && mkdir -p `pwd`/agent-3
-docker run \
-  --rm \
-  --net=host \
-  --name agent-3 \
-  --mount type=bind,source=`pwd`/agent-3,destination=/agent-3 \
-  gcr.io/etcd-development/etcd-functional-tester:go1.9.3 \
-  /bin/bash -c "/etcd-agent \
-    --etcd-path ${ETCD_EXEC_PATH} \
-    --etcd-log-dir /agent-3 \
-    --port :39027 \
-    --failpoint-addr :7383"

tools/functional-tester/scripts/docker-local-agent.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+<<COMMENT
+# run 3 agents for 3-node local etcd cluster
+./scripts/docker-local-agent.sh 1
+./scripts/docker-local-agent.sh 2
+./scripts/docker-local-agent.sh 3
+COMMENT
+
+if ! [[ "${0}" =~ "scripts/docker-local-agent.sh" ]]; then
+  echo "must be run from tools/functional-tester"
+  exit 255
+fi
+
+if [[ -z "${GO_VERSION}" ]]; then
+  GO_VERSION=1.9.4
+fi
+echo "Running with GO_VERSION:" ${GO_VERSION}
+
+if [[ -z ${1} ]]; then
+  echo "Expected second argument: 1, 2, or 3"
+  exit 255
+else
+  case ${1} in
+    1) ;;
+    2) ;;
+    3) ;;
+    *) echo "Expected second argument 1, 2, or 3, got" \"${1}\"
+       exit 255 ;;
+  esac
+  AGENT_NAME="agent-${1}"
+  AGENT_PORT_FLAG="--port :${1}9027"
+  FAILPOINT_ADDR_FLAG="--failpoint-addr :738${1}"
+fi
+echo "AGENT_NAME:" ${AGENT_NAME}
+echo "AGENT_PORT_FLAG:" ${AGENT_PORT_FLAG}
+echo "FAILPOINT_ADDR_FLAG:" ${FAILPOINT_ADDR_FLAG}
+
+if [[ -z "${ETCD_EXEC_PATH}" ]]; then
+  ETCD_EXEC_PATH=/etcd
+elif [[ "${ETCD_EXEC_PATH}" != "/etcd-failpoints" ]]; then
+  echo "Cannot find etcd executable:" ${ETCD_EXEC_PATH}
+  exit 255
+fi
+echo "ETCD_EXEC_PATH:" ${ETCD_EXEC_PATH}
+
+rm -rf `pwd`/${AGENT_NAME} && mkdir -p `pwd`/${AGENT_NAME}
+docker run \
+  --rm \
+  --net=host \
+  --name ${AGENT_NAME} \
+  --mount type=bind,source=`pwd`/${AGENT_NAME},destination=/${AGENT_NAME} \
+  gcr.io/etcd-development/etcd-functional-tester:go${GO_VERSION} \
+  /bin/bash -c "/etcd-agent \
+    --etcd-path ${ETCD_EXEC_PATH} \
+    --etcd-log-dir /${AGENT_NAME} \
+    ${AGENT_PORT_FLAG} \
+    ${FAILPOINT_ADDR_FLAG}"

tools/functional-tester/scripts/docker-local-tester.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+<<COMMENT
+# to run with different Go version
+# requires prebuilt Docker image
+#   GO_VERSION=1.9.4 make build-docker-functional-tester -f ./hack/scripts-dev/Makefile
+GO_VERSION=1.9.4 ./scripts/docker-local-tester.sh
+
+# to run only 1 tester round
+LIMIT=1 ./scripts/docker-local-tester.sh
+
+# to run long-running tests with no limit
+LIMIT=1 ./scripts/docker-local-tester.sh
+
+# to run only 1 tester round with election runner and others
+# default is STRESSER="keys,lease"
+LIMIT=1 \
+  STRESSER="keys,lease,election-runner,watch-runner,lock-racer-runner,lease-runner" \
+  ./scripts/docker-local-tester.sh
+
+# TODO: make stresser QPS configurable
+COMMENT
+
+if ! [[ "${0}" =~ "scripts/docker-local-tester.sh" ]]; then
+  echo "must be run from tools/functional-tester"
+  exit 255
+fi
+
+if [[ -z "${GO_VERSION}" ]]; then
+  GO_VERSION=1.9.4
+fi
+echo "Running with GO_VERSION:" ${GO_VERSION}
+
+if [[ "${LIMIT}" ]]; then
+  LIMIT_FLAG="--limit ${LIMIT}"
+  echo "Running with:" ${LIMIT_FLAG}
+else
+  echo "Running with no limit"
+fi
+
+if [[ "${STRESSER}" ]]; then
+  STRESSER_FLAG="--stresser ${STRESSER}"
+else
+  STRESSER_FLAG="--stresser keys,lease"
+fi
+echo "Running with:" ${STRESSER_FLAG}
+
+docker run \
+  --rm \
+  --net=host \
+  --name tester \
+  gcr.io/etcd-development/etcd-functional-tester:go${GO_VERSION} \
+  /bin/bash -c "/etcd-tester \
+    --agent-endpoints '127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027' \
+    --client-ports 1379,2379,3379 \
+    --advertise-client-ports 13790,23790,33790 \
+    --peer-ports 1380,2380,3380 \
+    --advertise-peer-ports 13800,23800,33800 \
+    ${LIMIT_FLAG} \
+    --etcd-runner /etcd-runner \
+    --stress-qps=2500 \
+    --stress-key-txn-count 100 \
+    --stress-key-txn-ops 10 \
+    ${STRESSER_FLAG} \
+    --exit-on-failure"

tools/functional-tester/scripts/tester-limit.sh

@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-
-if ! [[ "$0" =~ "scripts/tester-limit.sh" ]]; then
-  echo "must be run from tools/functional-tester"
-  exit 255
-fi
-
-# to run only 1 test round
-docker run \
-  --rm \
-  --net=host \
-  --name tester \
-  gcr.io/etcd-development/etcd-functional-tester:go1.9.3 \
-  /bin/bash -c "/etcd-tester \
-    --agent-endpoints '127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027' \
-    --client-ports 1379,2379,3379 \
-    --advertise-client-ports 13790,23790,33790 \
-    --peer-ports 1380,2380,3380 \
-    --advertise-peer-ports 13800,23800,33800 \
-    --limit 1 \
-    --stress-qps=2500 \
-    --stress-key-txn-count 100 \
-    --stress-key-txn-ops 10 \
-    --exit-on-failure"

tools/functional-tester/scripts/tester-runner.sh

@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-
-if ! [[ "$0" =~ "scripts/tester-runner.sh" ]]; then
-  echo "must be run from tools/functional-tester"
-  exit 255
-fi
-
-# to run with etcd-runner
-docker run \
-  --rm \
-  --net=host \
-  --name tester \
-  gcr.io/etcd-development/etcd-functional-tester:go1.9.3 \
-  /bin/bash -c "/etcd-tester \
-    --agent-endpoints '127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027' \
-    --client-ports 1379,2379,3379 \
-    --advertise-client-ports 13790,23790,33790 \
-    --peer-ports 1380,2380,3380 \
-    --advertise-peer-ports 13800,23800,33800 \
-    --stress-qps=2500 \
-    --stress-key-txn-count 100 \
-    --stress-key-txn-ops 10 \
-    --etcd-runner /etcd-runner \
-    --stresser=keys,lease,election-runner,watch-runner,lock-racer-runner,lease-runner \
-    --exit-on-failure"

tools/functional-tester/scripts/tester.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-if ! [[ "$0" =~ "scripts/tester.sh" ]]; then
-  echo "must be run from tools/functional-tester"
-  exit 255
-fi
-
-docker run \
-  --rm \
-  --net=host \
-  --name tester \
-  gcr.io/etcd-development/etcd-functional-tester:go1.9.3 \
-  /bin/bash -c "/etcd-tester \
-    --agent-endpoints '127.0.0.1:19027,127.0.0.1:29027,127.0.0.1:39027' \
-    --client-ports 1379,2379,3379 \
-    --advertise-client-ports 13790,23790,33790 \
-    --peer-ports 1380,2380,3380 \
-    --advertise-peer-ports 13800,23800,33800 \
-    --stress-qps=2500 \
-    --stress-key-txn-count 100 \
-    --stress-key-txn-ops 10 \
-    --exit-on-failure"