Migrates e2e tests from ginkgo,gomega to testing,testify

.github/workflows/commit.yaml

@@ -18,7 +18,7 @@ on:
 
 jobs:
   bin:
-    name: "Build `getenvoy` and `e2e` binaries for use in e2e tests"
+    name: "Build the `getenvoy` binary for use in e2e tests"
     runs-on: ubuntu-latest
     timeout-minutes: 15  # instead of 360 by default
     steps:
@@ -33,10 +33,10 @@ jobs:
       - name: "Init on first use"
         run: make init
 
-      - name: "Build `getenvoy` and `e2e` binaries"
+      - name: "Build the `getenvoy` binary"
         run: make bin
 
-      - name: "Share `getenvoy` and `e2e` binaries with the downstream jobs"
+      - name: "Share the `getenvoy` binary with the downstream jobs"
         uses: actions/upload-artifact@v2
         with:
           name: bin
@@ -52,7 +52,7 @@ jobs:
       - name: "Checkout"
         uses: actions/checkout@v2
 
-      - name: "Re-use `getenvoy` and `e2e` binaries pre-built by the upstream job"
+      - name: "Re-use the `getenvoy` binary pre-built by the upstream job"
         uses: actions/download-artifact@v2
         with:
           name: bin
@@ -61,8 +61,10 @@ jobs:
       - name: "Build language-specific Docker build images"
         run: make builders
 
-      - name: "Run e2e tests using `getenvoy` and `e2e` binaries built by the upstream job"
-        run: ./ci/e2e/linux/run_tests.sh
+      - name: "Run e2e tests using the `getenvoy` binary built by the upstream job"
+        run: |  # chmod to restore permissions lost in actions/download-artifact@v2
+          chmod a+x build/bin/linux/amd64/getenvoy
+          ./ci/e2e/macos/run_tests.sh
 
   e2e_macos:
     name: "Run e2e tests on MacOS"
@@ -74,7 +76,7 @@ jobs:
       - name: "Checkout"
         uses: actions/checkout@v2
 
-      - name: "Re-use `getenvoy` and `e2e` binaries pre-built by the upstream job"
+      - name: "Re-use the `getenvoy` binary pre-built by the upstream job"
         uses: actions/download-artifact@v2
         with:
           name: bin
@@ -92,5 +94,7 @@ jobs:
         run: make builders
         timeout-minutes: 10  # fail fast if MacOS runner becomes to slow
 
-      - name: "Run e2e tests using `getenvoy` and `e2e` binaries built by the upstream job"
-        run: ./ci/e2e/macos/run_tests.sh
+      - name: "Run e2e tests using the `getenvoy` binary built by the upstream job"
+        run: |  # chmod to restore permissions lost in actions/download-artifact@v2
+          chmod a+x build/bin/linux/amd64/getenvoy
+          ./ci/e2e/macos/run_tests.sh

.github/workflows/release.yaml

@@ -71,33 +71,11 @@ jobs:
       - name: "Push extension builder images"
         run: make builders.push BUILDERS_TAG=${{ env.RELEASE_VERSION }}
 
-  e2e_bin:
-    name: "Build `e2e` binaries for use in e2e tests"
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Checkout"
-        uses: actions/checkout@v2
-
-      - name: "Install Go"
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.16.2'
-
-      - name: "Build `e2e` binaries"
-        run: make build/bin/linux/amd64/e2e build/bin/darwin/amd64/e2e
-
-      - name: "Share `e2e` binaries with the downstream jobs"
-        uses: actions/upload-artifact@v2
-        with:
-          name: bin
-          path: build/bin
-
   e2e_linux:
     name: "Run e2e tests on Linux"
     needs:
       - getenvoy
       - builders
-      - e2e_bin
     runs-on: ubuntu-latest
     timeout-minutes: 30  # instead of 360 by default
     steps:
@@ -121,7 +99,7 @@ jobs:
         env:
           INPUT_FILE: getenvoy_${{ env.RELEASE_VERSION }}_Linux_x86_64.tar.gz
           INPUT_VERSION: tags/${{ env.RELEASE_TAG }}
-        run: |
+        run: |  # extract getenvoy to location used in `make e2e`. don't chmod because tar.gz should be correct.
           curl -s https://raw.githubusercontent.com/dsaltares/fetch-gh-release-asset/0.0.5/fetch_github_asset.sh | bash
           mkdir -p build/bin/linux/amd64
           tar -C build/bin/linux/amd64 -xf ${INPUT_FILE} getenvoy
@@ -138,7 +116,6 @@ jobs:
     needs:
       - getenvoy
       - builders
-      - e2e_bin
     runs-on: macos-latest
     timeout-minutes: 90  # instead of 360 by default
     steps:
@@ -162,7 +139,7 @@ jobs:
         env:
           INPUT_FILE: getenvoy_${{ env.RELEASE_VERSION }}_Darwin_x86_64.tar.gz
           INPUT_VERSION: tags/${{ env.RELEASE_TAG }}
-        run: |
+        run: |  # extract getenvoy to location used in `make e2e`. don't chmod because tar.gz should be correct.
           curl -s https://raw.githubusercontent.com/dsaltares/fetch-gh-release-asset/0.0.5/fetch_github_asset.sh | bash
           mkdir -p build/bin/darwin/amd64
           tar -C build/bin/darwin/amd64 -xf ${INPUT_FILE} getenvoy

Makefile

@@ -53,7 +53,10 @@ COVERAGE_PKG_LIST ?= $(shell go list ./pkg/... | grep -v -e github.com/tetratela
 GO_COVERAGE_OPTS ?= -covermode=atomic -coverpkg=./...
 GO_COVERAGE_EXTRA_OPTS ?=
 
-E2E_OPTS ?= -ginkgo.v
+E2E_PKG_LIST ?= ./test/e2e/...
+# Set the default timeout >10m as particularly rust e2e tests are slow https://golang.org/cmd/go/#hdr-Testing_flags
+# Run only one test at a time, in verbose mode, so that failures are easy to diagnose. Stop at first error.
+E2E_OPTS ?= -timeout 45m -test.parallel 1 -v -test.failfast
 E2E_EXTRA_OPTS ?=
 
 GOOSES := linux darwin
@@ -68,15 +71,6 @@ $(call GETENVOY_OUT_PATH,$(1),$(2)): generate
 endef
 $(foreach os,$(GOOSES),$(foreach arch,$(GOARCHS),$(eval $(call GEN_GETENVOY_BUILD_TARGET,$(os),$(arch)))))
 
-E2E_OUT_PATH = $(BIN_DIR)/$(1)/$(2)/e2e
-
-define GEN_E2E_BUILD_TARGET
-.PHONY: $(call E2E_OUT_PATH,$(1),$(2))
-$(call E2E_OUT_PATH,$(1),$(2)):
-	CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go test -c -o $(call E2E_OUT_PATH,$(1),$(2)) ./test/e2e
-endef
-$(foreach os,$(GOOSES),$(foreach arch,$(GOARCHS),$(eval $(call GEN_E2E_BUILD_TARGET,$(os),$(arch)))))
-
 .PHONY: init
 init: generate
 
@@ -109,9 +103,9 @@ test.ci: generate
 	go test $(GO_TEST_OPTS) $(GO_TEST_EXTRA_OPTS) $(TEST_PKG_LIST)
 
 .PHONY: e2e
-e2e: $(call GETENVOY_OUT_PATH,$(GOOS),$(GOARCH)) $(call E2E_OUT_PATH,$(GOOS),$(GOARCH))
+e2e: $(call GETENVOY_OUT_PATH,$(GOOS),$(GOARCH))
 	docker-compose up -d
-	E2E_GETENVOY_BINARY=$(PWD)/$(call GETENVOY_OUT_PATH,$(GOOS),$(GOARCH)) $(call E2E_OUT_PATH,$(GOOS),$(GOARCH)) $(GO_TEST_OPTS) $(GO_TEST_EXTRA_OPTS) $(E2E_OPTS) $(E2E_EXTRA_OPTS)
+	E2E_GETENVOY_BINARY=$(PWD)/$(call GETENVOY_OUT_PATH,$(GOOS),$(GOARCH)) go test github.com/tetratelabs/getenvoy/test/e2e $(E2E_OPTS) $(E2E_EXTRA_OPTS) $(E2E_PKG_LIST)
 
 .PHONY: bin
 bin: $(foreach os,$(GOOSES), bin/$(os))
@@ -124,7 +118,7 @@ $(foreach os,$(GOOSES),$(eval $(call GEN_BIN_GOOS_TARGET,$(os))))
 
 define GEN_BIN_GOOS_GOARCH_TARGET
 .PHONY: bin/$(1)/$(2)
-bin/$(1)/$(2): $(call GETENVOY_OUT_PATH,$(1),$(2)) $(call E2E_OUT_PATH,$(1),$(2))
+bin/$(1)/$(2): $(call GETENVOY_OUT_PATH,$(1),$(2))
 endef
 $(foreach os,$(GOOSES),$(foreach arch,$(GOARCHS),$(eval $(call GEN_BIN_GOOS_GOARCH_TARGET,$(os),$(arch)))))
 

ci/e2e/linux/run_tests.sh

@@ -16,9 +16,8 @@
 
 set -e
 
-SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}")"  && pwd)
-WORKSPACE_DIR="${SCRIPT_DIR}/../../.."
-
+# see #153 as some of the below may be incorporated into make or CI setup or otherwise improved
+# TODO: why default to /tmp while macos defaults to $HOME?
 E2E_CACHE_DIR="${E2E_CACHE_DIR:-/tmp/cache/getenvoy}"
 
 # make sure the cache directory is first created on behalf of the current user
@@ -29,14 +28,5 @@ sudo chown -R $(id -u):$(id -g) "${E2E_CACHE_DIR}"
 # to speed up `getenvoy extension build|test`, re-use a single cache across all extensions created by e2e tests
 export E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS} -v ${E2E_CACHE_DIR}:/tmp/cache/getenvoy -e CARGO_HOME=/tmp/cache/getenvoy/extension/rust-builder/cargo"
 
-# restore executable bit that get lost by Github Actions during artifact upload/download
-chmod a+x ${WORKSPACE_DIR}/build/bin/linux/amd64/*
-
-# start other containers required in e2e tests
-docker-compose up -d
-
-# run e2e tests on a `getenvoy` binary built by the upstream job
-export E2E_GETENVOY_BINARY="${WORKSPACE_DIR}/build/bin/linux/amd64/getenvoy"
-
-# run e2e tests with '-ginkgo.v' flag to be able to see the progress
-${WORKSPACE_DIR}/build/bin/linux/amd64/e2e -ginkgo.v
+# run the normal make script.
+make e2e

ci/e2e/macos/run_tests.sh

@@ -16,9 +16,8 @@
 
 set -e
 
-SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}")"  && pwd)
-WORKSPACE_DIR="${SCRIPT_DIR}/../../.."
-
+# see #153 as some of the below may be incorporated into make or CI setup or otherwise improved
+# TODO: why default to HOME where linux defaults to /tmp?
 E2E_CACHE_DIR="${E2E_CACHE_DIR:-$HOME/cache/getenvoy}"
 
 # make sure the cache directory is first created on behalf of the current user
@@ -28,17 +27,8 @@ mkdir -p "${E2E_CACHE_DIR}"
 # to speed up `getenvoy extension build|test`, re-use a single cache across all extensions created by e2e tests
 export E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS} -v ${E2E_CACHE_DIR}:/tmp/cache/getenvoy -e CARGO_HOME=/tmp/cache/getenvoy/extension/rust-builder/cargo"
 
-# set HOME directory
+# set HOME directory (TODO: why? why not GETENVOY_HOME? why not also in linux?)
 export E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS} -e HOME=/tmp/getenvoy"
 
-# restore executable bit that get lost by Github Actions during artifact upload/download
-chmod a+x ${WORKSPACE_DIR}/build/bin/darwin/amd64/*
-
-# start other containers required in e2e tests
-docker-compose up -d
-
-# run e2e tests on a `getenvoy` binary built by the upstream job
-export E2E_GETENVOY_BINARY="${WORKSPACE_DIR}/build/bin/darwin/amd64/getenvoy"
-
-# run e2e tests with '-ginkgo.v' flag to be able to see the progress
-${WORKSPACE_DIR}/build/bin/darwin/amd64/e2e -ginkgo.v
+# run the normal make script.
+make e2e