#!/usr/bin/env bash # # Run all etcd tests # ./test # ./test -v # # # Run specified test pass # # $ PASSES=unit ./test # $ PASSES=integration ./test # # # Run tests for one package # Each pass has different default timeout, if you just run tests in one package or 1 test case then you can set TIMEOUT # flag for different expectation # # $ PASSES=unit PKG=./wal TIMEOUT=1m ./test # $ PASSES=integration PKG=client/integration TIMEOUT=1m ./test # # # Run specified unit tests in one package # To run all the tests with prefix of "TestNew", set "TESTCASE=TestNew "; # to run only "TestNew", set "TESTCASE="\bTestNew\b"" # # $ PASSES=unit PKG=./wal TESTCASE=TestNew TIMEOUT=1m ./test # $ PASSES=unit PKG=./wal TESTCASE="\bTestNew\b" TIMEOUT=1m ./test # $ PASSES=integration PKG=client/integration TESTCASE="\bTestV2NoRetryEOF\b" TIMEOUT=1m ./test # # # Run code coverage # COVERDIR must either be a absolute path or a relative path to the etcd root # $ COVERDIR=coverage PASSES="build_cov cov" ./test set -e source ./build # build before setting up test GOPATH if [[ "${PASSES}" == *"functional"* ]]; then ./functional/build fi if [ -z "$PASSES" ]; then PASSES="fmt bom dep build unit" fi USERPKG=${PKG:-} # Invoke ./tests/cover.test.bash for HTML output COVER=${COVER:-"-cover"} # Hack: gofmt ./ will recursively check the .git directory. So use *.go for gofmt. IGNORE_PKGS="(vendor/|etcdserverpb|rafttest|gopath.proto|v3lockpb|v3electionpb)" INTEGRATION_PKGS="(integration|tests/e2e|contrib|functional)" # all github.com/etcd-io/etcd/whatever pkgs that are not auto-generated / tools # shellcheck disable=SC1117 PKGS=$(find . -name \*.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | grep -vE "(tools/|contrib/|tests/e2e|pb)" | sed "s|\.|${REPO_PATH}|g" | xargs echo) # pkg1,pkg2,pkg3 PKGS_COMMA=${PKGS// /,} # shellcheck disable=SC1117 TEST_PKGS=$(find . -name \*_test.go | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g") # shellcheck disable=SC1117 FORMATTABLE=$(find . -name \*.go | while read -r a; do echo "$(dirname "$a")/*.go"; done | sort | uniq | grep -vE "$IGNORE_PKGS" | sed "s|\./||g") TESTABLE_AND_FORMATTABLE=$(echo "$TEST_PKGS" | grep -vE "$INTEGRATION_PKGS") # check if user provided PKG override if [ -z "${USERPKG}" ]; then TEST=$TESTABLE_AND_FORMATTABLE FMT=$FORMATTABLE else # strip out leading dotslashes and trailing slashes from PKG=./foo/ TEST=${USERPKG/#./} TEST=${TEST/#\//} TEST=${TEST/%\//} # only run gofmt on packages provided by user FMT="$TEST" fi # shellcheck disable=SC2206 FMT=($FMT) if [ "${VERBOSE}" == "1" ]; then # shellcheck disable=SC2128 echo "Running with FMT:" "${FMT[@]}" fi # prepend REPO_PATH to each local package split=$TEST TEST="" for a in $split; do TEST="$TEST ${REPO_PATH}/${a}"; done # shellcheck disable=SC2206 TEST=($TEST) if [ "${VERBOSE}" == "1" ]; then # shellcheck disable=SC2128 echo "Running with TEST:" "${TEST[@]}" fi # TODO: 'rafttest' is failing with unused STATIC_ANALYSIS_PATHS=$(find . -name \*.go ! -path './vendor/*' ! -path './gopath.proto/*' ! -path '*pb/*' | while read -r a; do dirname "$a"; done | sort | uniq | grep -vE "$IGNORE_PKGS") # shellcheck disable=SC2206 STATIC_ANALYSIS_PATHS=($STATIC_ANALYSIS_PATHS) if [ "${VERBOSE}" == "1" ]; then # shellcheck disable=SC2128 echo "Running with STATIC_ANALYSIS_PATHS:" "${STATIC_ANALYSIS_PATHS[@]}" fi if [ -z "$GOARCH" ]; then GOARCH=$(go env GOARCH); fi # determine the number of CPUs to use for Go tests TEST_CPUS="1,2,4" if [ -n "${CPU}" ]; then TEST_CPUS="${CPU}" fi echo "Running with TEST_CPUS:" "${TEST_CPUS}" # determine whether target supports race detection if [ "$GOARCH" == "amd64" ]; then RACE="--race" fi RUN_ARG="" if [ -n "${TESTCASE}" ]; then RUN_ARG="-run=${TESTCASE}" fi function unit_pass { echo "Running unit tests..." GO_TEST_FLAG="" if [ "${VERBOSE}" == "1" ]; then GO_TEST_FLAG="-v" fi if [ "${VERBOSE}" == "2" ]; then GO_TEST_FLAG="-v" export CLIENT_DEBUG=1 fi if [ "${RUN_ARG}" == "" ]; then RUN_ARG="-run=Test" fi # check if user provided time out, especially useful when just run one test case # expectation could be different USERTIMEOUT="" if [ -z "${TIMEOUT}" ]; then USERTIMEOUT="3m" else USERTIMEOUT="${TIMEOUT}" fi go test ${GO_TEST_FLAG} -timeout "${USERTIMEOUT}" "${COVER}" ${RACE} -cpu "${TEST_CPUS}" ${RUN_ARG} "$@" "${TEST[@]}" } function integration_pass { echo "Running integration tests..." # check if user provided time out, especially useful when just run one test case # expectation could be different USERTIMEOUT="" if [ -z "${TIMEOUT}" ]; then USERTIMEOUT="30m" else USERTIMEOUT="${TIMEOUT}" fi # if TESTCASE and PKG set, run specified test case in specified PKG # if TESTCASE set, PKG not set, run specified test case in all integration and integration_extra packages # if TESTCASE not set, PKG set, run all test cases in specified package # if TESTCASE not set, PKG not set, run all tests in all integration and integration_extra packages if [ -z "${TESTCASE}" ] && [ -z "${USERPKG}" ]; then go test -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration" integration_extra "$@" else if [ -z "${USERPKG}" ]; then INTEGTESTPKG=("${REPO_PATH}/integration" "${REPO_PATH}/client/integration" "${REPO_PATH}/clientv3/integration" "${REPO_PATH}/contrib/raftexample" "${REPO_PATH}/store") else INTEGTESTPKG=("${TEST[@]}") fi go test -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "${RUN_ARG}" "$@" "${INTEGTESTPKG[@]}" fi } function integration_extra { go test -timeout 1m -v ${RACE} -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/client/integration" go test -timeout 25m -v ${RACE} -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/clientv3/integration" go test -timeout 1m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/contrib/raftexample" go test -timeout 5m -v ${RACE} -tags v2v3 "$@" "${REPO_PATH}/etcdserver/api/v2store" go test -timeout 1m -v ${RACE} -cpu "${TEST_CPUS}" -run=Example "$@" "${TEST[@]}" } function functional_pass { # Clean up any data and logs from previous runs rm -rf /tmp/etcd-functional-* /tmp/etcd-functional-*.backup for a in 1 2 3; do ./bin/etcd-agent --network tcp --address 127.0.0.1:${a}9027 & pid="$!" agent_pids="${agent_pids} $pid" done for a in 1 2 3; do echo "Waiting for 'etcd-agent' on ${a}9027..." while ! nc -z localhost ${a}9027; do sleep 1 done done echo "functional test START!" ./bin/etcd-tester --config ./functional.yaml && echo "'etcd-tester' succeeded" ETCD_TESTER_EXIT_CODE=$? echo "ETCD_TESTER_EXIT_CODE:" ${ETCD_TESTER_EXIT_CODE} # shellcheck disable=SC2206 agent_pids=($agent_pids) kill -s TERM "${agent_pids[@]}" || true if [[ "${ETCD_TESTER_EXIT_CODE}" -ne "0" ]]; then printf "\n" echo "FAILED! 'tail -1000 /tmp/etcd-functional-1/etcd.log'" tail -1000 /tmp/etcd-functional-1/etcd.log printf "\n" echo "FAILED! 'tail -1000 /tmp/etcd-functional-2/etcd.log'" tail -1000 /tmp/etcd-functional-2/etcd.log printf "\n" echo "FAILED! 'tail -1000 /tmp/etcd-functional-3/etcd.log'" tail -1000 /tmp/etcd-functional-3/etcd.log echo "--- FAIL: exit code" ${ETCD_TESTER_EXIT_CODE} exit ${ETCD_TESTER_EXIT_CODE} fi echo "functional test PASS!" } function cov_pass { echo "Running code coverage..." # install gocovmerge before running code coverage from github.com/wadey/gocovmerge # gocovmerge merges coverage files if ! command -v gocovmerge >/dev/null; then echo "gocovmerge not installed" exit 255 fi if [ -z "$COVERDIR" ]; then echo "COVERDIR undeclared" exit 255 fi if [ ! -f "bin/etcd_test" ]; then echo "etcd_test binary not found" exit 255 fi mkdir -p "$COVERDIR" # run code coverage for unit and integration tests GOCOVFLAGS="-covermode=set -coverpkg ${PKGS_COMMA} -v -timeout 30m" # shellcheck disable=SC2206 GOCOVFLAGS=($GOCOVFLAGS) failed="" for t in $(echo "${TEST_PKGS}" | grep -vE "(tests/e2e|functional)"); do tf=$(echo "$t" | tr / _) # cache package compilation data for faster repeated builds go test "${GOCOVFLAGS[@]}" -i "${REPO_PATH}/$t" || true # uses -run=Test to skip examples because clientv3/ example tests will leak goroutines go test "${GOCOVFLAGS[@]}" -run=Test -coverprofile "$COVERDIR/${tf}.coverprofile" "${REPO_PATH}/$t" || failed="$failed $t" done # v2v3 tests go test -tags v2v3 "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/store-v2v3.coverprofile" "${REPO_PATH}/clientv3/integration" || failed="$failed store-v2v3" # proxy tests go test -tags cluster_proxy "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/proxy_integration.coverprofile" "${REPO_PATH}/integration" || failed="$failed proxy-integration" go test -tags cluster_proxy "${GOCOVFLAGS[@]}" -coverprofile "$COVERDIR/proxy_clientv3.coverprofile" "${REPO_PATH}/clientv3/integration" || failed="$failed proxy-clientv3/integration" # run code coverage for e2e tests # use 30m timeout because e2e coverage takes longer # due to many tests cause etcd process to wait # on leadership transfer timeout during gracefully shutdown echo Testing tests/e2e without proxy... go test -tags cov -timeout 30m -v "${REPO_PATH}/tests/e2e" || failed="$failed tests/e2e" echo Testing tests/e2e with proxy... go test -tags "cov cluster_proxy" -timeout 30m -v "${REPO_PATH}/tests/e2e" || failed="$failed tests/e2e-proxy" # incrementally merge to get coverage data even if some coverage files are corrupted # optimistically assume etcdserver package's coverage file is OK since gocovmerge # expects to start with a non-empty file cp "$COVERDIR"/etcdserver.coverprofile "$COVERDIR"/cover.out for f in "$COVERDIR"/*.coverprofile; do echo "merging test coverage file ${f}" gocovmerge "$f" "$COVERDIR"/cover.out >"$COVERDIR"/cover.tmp || failed="$failed $f" if [ -s "$COVERDIR"/cover.tmp ]; then mv "$COVERDIR"/cover.tmp "$COVERDIR"/cover.out fi done # strip out generated files (using GNU-style sed) sed --in-place '/generated.go/d' "$COVERDIR"/cover.out || true # held failures to generate the full coverage file, now fail if [ -n "$failed" ]; then for f in $failed; do echo "--- FAIL:" "$f" done exit 255 fi } function e2e_pass { echo "Running e2e tests..." # check if user provided time out, especially useful when just run one test case # expectation could be different USERTIMEOUT="" if [ -z "${TIMEOUT}" ]; then USERTIMEOUT="30m" else USERTIMEOUT="${TIMEOUT}" fi go test -timeout "${USERTIMEOUT}" -v -cpu "${TEST_CPUS}" "${RUN_ARG}" "$@" "${REPO_PATH}/tests/e2e" } function integration_e2e_pass { echo "Running integration and e2e tests..." go test -timeout 30m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/tests/e2e" & e2epid="$!" go test -timeout 30m -v -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration" & intpid="$!" wait $e2epid wait $intpid integration_extra "$@" } function grpcproxy_pass { go test -timeout 30m -v ${RACE} -tags cluster_proxy -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/integration" go test -timeout 30m -v ${RACE} -tags cluster_proxy -cpu "${TEST_CPUS}" "$@" "${REPO_PATH}/clientv3/integration" go test -timeout 30m -v -tags cluster_proxy "$@" "${REPO_PATH}/tests/e2e" } function release_pass { rm -f ./bin/etcd-last-release # to grab latest patch release; bump this up for every minor release UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.3.*" | head -1) if [ -n "$MANUAL_VER" ]; then # in case, we need to test against different version UPGRADE_VER=$MANUAL_VER fi if [[ -z ${UPGRADE_VER} ]]; then UPGRADE_VER="v3.3.0" echo "fallback to" ${UPGRADE_VER} fi local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz" echo "Downloading $file" set +e curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file" local result=$? set -e case $result in 0) ;; *) echo "--- FAIL:" ${result} exit $result ;; esac tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1 mkdir -p ./bin mv /tmp/etcd ./bin/etcd-last-release } function shellcheck_pass { if command -v shellcheck >/dev/null; then shellcheckResult=$(shellcheck -fgcc build test scripts/*.sh 2>&1 || true) if [ -n "${shellcheckResult}" ]; then echo -e "shellcheck checking failed:\\n${shellcheckResult}" exit 255 fi fi } function markdown_you_pass { # eschew you yous=$(find . -name \*.md ! -path './vendor/*' ! -path './Documentation/v2/*' ! -path './gopath.proto/*' -exec grep -E --color "[Yy]ou[r]?[ '.,;]" {} + || true) if [ -n "$yous" ]; then echo -e "found 'you' in documentation:\\n${yous}" exit 255 fi } function markdown_marker_pass { # TODO: check other markdown files when marker handles headers with '[]' if command -v marker >/dev/null; then markerResult=$(marker --skip-http --root ./Documentation 2>&1 || true) if [ -n "${markerResult}" ]; then echo -e "marker checking failed:\\n${markerResult}" exit 255 fi else echo "Skipping marker..." fi } function goword_pass { if command -v goword >/dev/null; then # get all go files to process gofiles=$(find "${FMT[@]}" -iname '*.go' 2>/dev/null) # shellcheck disable=SC2206 gofiles_all=($gofiles) # ignore tests and protobuf files # shellcheck disable=SC1117 gofiles=$(echo "${gofiles_all[@]}" | sort | uniq | sed "s/ /\n/g" | grep -vE "(\\_test.go|\\.pb\\.go)") # shellcheck disable=SC2206 gofiles=($gofiles) # only check for broken exported godocs gowordRes=$(goword -use-spell=false "${gofiles[@]}" | grep godoc-export | sort) if [ -n "$gowordRes" ]; then echo -e "goword checking failed:\\n${gowordRes}" exit 255 fi # check some spelling gowordRes=$(goword -ignore-file=.words clientv3/{*,*/*}.go 2>&1 | grep spell | sort) if [ -n "$gowordRes" ]; then echo -e "goword checking failed:\\n${gowordRes}" exit 255 fi else echo "Skipping goword..." fi } function gofmt_pass { fmtRes=$(gofmt -l -s -d "${FMT[@]}") if [ -n "${fmtRes}" ]; then echo -e "gofmt checking failed:\\n${fmtRes}" exit 255 fi } function govet_pass { vetRes=$(go vet "${TEST[@]}") if [ -n "${vetRes}" ]; then echo -e "govet checking failed:\\n${vetRes}" exit 255 fi } function govet_shadow_pass { fmtpkgs=$(for a in "${FMT[@]}"; do dirname "$a"; done | sort | uniq | grep -v "\\.") # shellcheck disable=SC2206 fmtpkgs=($fmtpkgs) # Golang 1.12 onwards the experimental -shadow option is no longer available with go vet go get golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow export PATH=${GOPATH}/bin:${PATH} shadow_tool=$(which shadow) vetRes=$(go vet -all -vettool="${shadow_tool}" "${TEST[@]}") if [ -n "${vetRes}" ]; then echo -e "govet -all -shadow checking failed:\\n${vetRes}" exit 255 fi } function unparam_pass { if command -v unparam >/dev/null; then unparamResult=$(unparam "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true) if [ -n "${unparamResult}" ]; then echo -e "unparam checking failed:\\n${unparamResult}" exit 255 fi else echo "Skipping unparam..." fi } function staticcheck_pass { if command -v staticcheck >/dev/null; then staticcheckResult=$(staticcheck "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true) if [ -n "${staticcheckResult}" ]; then # TODO: resolve these after go1.8 migration # See https://github.com/dominikh/go-tools/tree/master/cmd/staticcheck STATIC_CHECK_MASK="S(A|T)(1002|1005|1006|1008|1012|1019|1032|2002|4003|4006)" if echo "${staticcheckResult}" | grep -vE "$STATIC_CHECK_MASK"; then echo -e "staticcheck checking failed:\\n${staticcheckResult}" exit 255 else suppressed=$(echo "${staticcheckResult}" | sed 's/ /\n/g' | grep "(SA" | sort | uniq -c) echo -e "staticcheck suppressed warnings:\\n${suppressed}" fi fi else echo "Skipping staticcheck..." fi } function revive_pass { if command -v revive >/dev/null; then reviveResult=$(revive -config ./tests/revive.toml -exclude "vendor/..." ./... 2>&1 || true) if [ -n "${reviveResult}" ]; then echo -e "revive checking failed:\\n${reviveResult}" exit 255 fi else echo "Skipping revive..." fi } function unconvert_pass { if command -v unconvert >/dev/null; then unconvertResult=$(unconvert -v "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true) if [ -n "${unconvertResult}" ]; then echo -e "unconvert checking failed:\\n${unconvertResult}" exit 255 fi else echo "Skipping unconvert..." fi } function ineffassign_pass { if command -v ineffassign >/dev/null; then ineffassignResult=$(ineffassign "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true) if [ -n "${ineffassignResult}" ]; then echo -e "ineffassign checking failed:\\n${ineffassignResult}" exit 255 fi else echo "Skipping ineffassign..." fi } function nakedret_pass { if command -v nakedret >/dev/null; then nakedretResult=$(nakedret "${STATIC_ANALYSIS_PATHS[@]}" 2>&1 || true) if [ -n "${nakedretResult}" ]; then echo -e "nakedret checking failed:\\n${nakedretResult}" exit 255 fi else echo "Skipping nakedret..." fi } function license_header_pass { licRes="" files=$(find . -type f -iname '*.go' ! -path './vendor/*' ! -path './gopath.proto/*') for file in $files; do if ! head -n3 "${file}" | grep -Eq "(Copyright|generated|GENERATED)" ; then licRes="${licRes}"$(echo -e " ${file}") fi done if [ -n "${licRes}" ]; then echo -e "license header checking failed:\\n${licRes}" exit 255 fi } function receiver_name_pass { # shellcheck disable=SC1117 recvs=$(grep 'func ([^*]' {*,*/*,*/*/*}.go | grep -Ev "(generated|pb/)" | tr ':' ' ' | \ awk ' { print $2" "$3" "$4" "$1 }' | sed "s/[a-zA-Z\.]*go//g" | sort | uniq | \ grep -Ev "(Descriptor|Proto|_)" | awk ' { print $3" "$4 } ' | sort | uniq -c | grep -v ' 1 ' | awk ' { print $2 } ') if [ -n "${recvs}" ]; then # shellcheck disable=SC2206 recvs=($recvs) for recv in "${recvs[@]}"; do echo "Mismatched receiver for $recv..." grep "$recv" "${FMT[@]}" | grep 'func (' done exit 255 fi } function commit_title_pass { git log --oneline "$(git merge-base HEAD master)"...HEAD | while read -r l; do commitMsg=$(echo "$l" | cut -f2- -d' ') if [[ "$commitMsg" == Merge* ]]; then # ignore "Merge pull" commits continue fi if [[ "$commitMsg" == Revert* ]]; then # ignore revert commits continue fi pkgPrefix=$(echo "$commitMsg" | cut -f1 -d':') spaceCommas=$(echo "$commitMsg" | sed 's/ /\n/g' | grep -c ',$' || echo 0) commaSpaces=$(echo "$commitMsg" | sed 's/,/\n/g' | grep -c '^ ' || echo 0) if [[ $(echo "$commitMsg" | grep -c ":..*") == 0 || "$commitMsg" == "$pkgPrefix" || "$spaceCommas" != "$commaSpaces" ]]; then echo "$l"... echo "Expected commit title format '{\", \"}: '" echo "Got: $l" exit 255 fi done } # tools gosimple,unused,staticheck,unconvert,ineffasign,nakedret # are not module-aware. See https://github.com/golang/go/issues/24661 # The module-aware versions need to be used when they become available function fmt_pass { toggle_failpoints disable # TODO: add "unparam" for p in shellcheck \ markdown_you \ markdown_marker \ goword \ gofmt \ govet \ govet_shadow \ revive \ license_header \ receiver_name \ commit_title \ ; do echo "'$p' started at $(date)" "${p}"_pass "$@" echo "'$p' completed at $(date)" done } function bom_pass { if ! command -v license-bill-of-materials >/dev/null; then return fi if [ "${GO111MODULE}" == "on" ]; then # license-bill-off-materials calls "go list std cmd" which cannot handle modules # Please see https://github.com/golang/go/issues/26924 echo "Skipping license-bill-of-materials with go modules..." return fi echo "Checking bill of materials..." license-bill-of-materials \ --override-file bill-of-materials.override.json \ go.etcd.io/etcd go.etcd.io/etcd/etcdctl >bom-now.json || true if ! diff bill-of-materials.json bom-now.json; then echo "vendored licenses do not match given bill of materials" exit 255 fi rm bom-now.json } function dep_pass { echo "Checking package dependencies..." # don't pull in etcdserver package pushd clientv3 >/dev/null badpkg="(etcdserver$|mvcc$|backend$|grpc-gateway)" deps=$(go list -f '{{ .Deps }}' | sed 's/ /\n/g' | grep -E "${badpkg}" || echo "") popd >/dev/null if [ -n "$deps" ]; then echo -e "clientv3 has masked dependencies:\\n${deps}" exit 255 fi } function build_cov_pass { out="bin" if [ -n "${BINDIR}" ]; then out="${BINDIR}"; fi go test -tags cov -c -covermode=set -coverpkg="$PKGS_COMMA" -o "${out}/etcd_test" go test -tags cov -c -covermode=set -coverpkg="$PKGS_COMMA" -o "${out}/etcdctl_test" "${REPO_PATH}/etcdctl" } # fail fast on static tests function build_pass { echo "Checking build..." GO_BUILD_FLAGS="-v" etcd_build GO_BUILD_FLAGS="-v" tools_build } for pass in $PASSES; do echo "Starting '$pass' pass at $(date)" "${pass}"_pass "$@" echo "Finished '$pass' pass at $(date)" done echo "Success"