From b24f531e17fa3a14c5e5e3dea1ab5038a71ed039 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Tue, 30 Mar 2021 15:33:52 +0800 Subject: [PATCH 1/8] Runs e2e directly instead of via ginkgo Signed-off-by: Adrian Cole --- .github/workflows/commit.yaml | 14 +- .github/workflows/release.yaml | 23 -- Makefile | 18 +- ci/e2e/linux/run_tests.sh | 4 +- ci/e2e/macos/run_tests.sh | 2 +- test/e2e/e2e_suite_test.go | 67 ----- test/e2e/e2e_test.go | 162 +++++++++++ test/e2e/getenvoy_extension_build_test.go | 95 +++---- test/e2e/getenvoy_extension_examples_test.go | 175 +++++------- test/e2e/getenvoy_extension_init_test.go | 153 +++++------ test/e2e/getenvoy_extension_push_test.go | 267 ++++++------------- test/e2e/getenvoy_extension_run_test.go | 199 +++++--------- test/e2e/getenvoy_extension_test_test.go | 84 +++--- test/e2e/getenvoy_test.go | 27 +- test/e2e/util/command.go | 53 ++-- test/e2e/util/iostream.go | 4 +- 16 files changed, 591 insertions(+), 756 deletions(-) delete mode 100644 test/e2e/e2e_suite_test.go create mode 100644 test/e2e/e2e_test.go diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml index 8a21df1f..8dacf911 100644 --- a/.github/workflows/commit.yaml +++ b/.github/workflows/commit.yaml @@ -18,7 +18,7 @@ on: jobs: bin: - name: "Build `getenvoy` and `e2e` binaries for use in e2e tests" + name: "Build the `getenvoy` binary for use in e2e tests" runs-on: ubuntu-latest timeout-minutes: 15 # instead of 360 by default steps: @@ -33,10 +33,10 @@ jobs: - name: "Init on first use" run: make init - - name: "Build `getenvoy` and `e2e` binaries" + - name: "Build the `getenvoy` binary" run: make bin - - name: "Share `getenvoy` and `e2e` binaries with the downstream jobs" + - name: "Share the `getenvoy` binary with the downstream jobs" uses: actions/upload-artifact@v2 with: name: bin @@ -52,7 +52,7 @@ jobs: - name: "Checkout" uses: actions/checkout@v2 - - name: "Re-use `getenvoy` and `e2e` binaries pre-built by the upstream job" + - name: "Re-use the `getenvoy` binary pre-built by the upstream job" uses: actions/download-artifact@v2 with: name: bin @@ -61,7 +61,7 @@ jobs: - name: "Build language-specific Docker build images" run: make builders - - name: "Run e2e tests using `getenvoy` and `e2e` binaries built by the upstream job" + - name: "Run e2e tests using the `getenvoy` binary built by the upstream job" run: ./ci/e2e/linux/run_tests.sh e2e_macos: @@ -74,7 +74,7 @@ jobs: - name: "Checkout" uses: actions/checkout@v2 - - name: "Re-use `getenvoy` and `e2e` binaries pre-built by the upstream job" + - name: "Re-use the `getenvoy` binary pre-built by the upstream job" uses: actions/download-artifact@v2 with: name: bin @@ -92,5 +92,5 @@ jobs: run: make builders timeout-minutes: 10 # fail fast if MacOS runner becomes to slow - - name: "Run e2e tests using `getenvoy` and `e2e` binaries built by the upstream job" + - name: "Run e2e tests using the `getenvoy` binary built by the upstream job" run: ./ci/e2e/macos/run_tests.sh diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 7f64fd91..54527e8b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -71,33 +71,11 @@ jobs: - name: "Push extension builder images" run: make builders.push BUILDERS_TAG=${{ env.RELEASE_VERSION }} - e2e_bin: - name: "Build `e2e` binaries for use in e2e tests" - runs-on: ubuntu-latest - steps: - - name: "Checkout" - uses: actions/checkout@v2 - - - name: "Install Go" - uses: actions/setup-go@v2 - with: - go-version: '1.16.2' - - - name: "Build `e2e` binaries" - run: make build/bin/linux/amd64/e2e build/bin/darwin/amd64/e2e - - - name: "Share `e2e` binaries with the downstream jobs" - uses: actions/upload-artifact@v2 - with: - name: bin - path: build/bin - e2e_linux: name: "Run e2e tests on Linux" needs: - getenvoy - builders - - e2e_bin runs-on: ubuntu-latest timeout-minutes: 30 # instead of 360 by default steps: @@ -138,7 +116,6 @@ jobs: needs: - getenvoy - builders - - e2e_bin runs-on: macos-latest timeout-minutes: 90 # instead of 360 by default steps: diff --git a/Makefile b/Makefile index 5979ba24..22c9ecf4 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,8 @@ COVERAGE_PKG_LIST ?= $(shell go list ./pkg/... | grep -v -e github.com/tetratela GO_COVERAGE_OPTS ?= -covermode=atomic -coverpkg=./... GO_COVERAGE_EXTRA_OPTS ?= -E2E_OPTS ?= -ginkgo.v +E2E_PKG_LIST ?= ./test/e2e/... +E2E_OPTS ?= -test.parallel 1 -test.failfast E2E_EXTRA_OPTS ?= GOOSES := linux darwin @@ -68,15 +69,6 @@ $(call GETENVOY_OUT_PATH,$(1),$(2)): generate endef $(foreach os,$(GOOSES),$(foreach arch,$(GOARCHS),$(eval $(call GEN_GETENVOY_BUILD_TARGET,$(os),$(arch))))) -E2E_OUT_PATH = $(BIN_DIR)/$(1)/$(2)/e2e - -define GEN_E2E_BUILD_TARGET -.PHONY: $(call E2E_OUT_PATH,$(1),$(2)) -$(call E2E_OUT_PATH,$(1),$(2)): - CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go test -c -o $(call E2E_OUT_PATH,$(1),$(2)) ./test/e2e -endef -$(foreach os,$(GOOSES),$(foreach arch,$(GOARCHS),$(eval $(call GEN_E2E_BUILD_TARGET,$(os),$(arch))))) - .PHONY: init init: generate @@ -109,9 +101,9 @@ test.ci: generate go test $(GO_TEST_OPTS) $(GO_TEST_EXTRA_OPTS) $(TEST_PKG_LIST) .PHONY: e2e -e2e: $(call GETENVOY_OUT_PATH,$(GOOS),$(GOARCH)) $(call E2E_OUT_PATH,$(GOOS),$(GOARCH)) +e2e: $(call GETENVOY_OUT_PATH,$(GOOS),$(GOARCH)) docker-compose up -d - E2E_GETENVOY_BINARY=$(PWD)/$(call GETENVOY_OUT_PATH,$(GOOS),$(GOARCH)) $(call E2E_OUT_PATH,$(GOOS),$(GOARCH)) $(GO_TEST_OPTS) $(GO_TEST_EXTRA_OPTS) $(E2E_OPTS) $(E2E_EXTRA_OPTS) + E2E_GETENVOY_BINARY=$(PWD)/$(call GETENVOY_OUT_PATH,$(GOOS),$(GOARCH)) go test github.com/tetratelabs/getenvoy/test/e2e $(E2E_OPTS) $(E2E_EXTRA_OPTS) $(E2E_PKG_LIST) .PHONY: bin bin: $(foreach os,$(GOOSES), bin/$(os)) @@ -124,7 +116,7 @@ $(foreach os,$(GOOSES),$(eval $(call GEN_BIN_GOOS_TARGET,$(os)))) define GEN_BIN_GOOS_GOARCH_TARGET .PHONY: bin/$(1)/$(2) -bin/$(1)/$(2): $(call GETENVOY_OUT_PATH,$(1),$(2)) $(call E2E_OUT_PATH,$(1),$(2)) +bin/$(1)/$(2): $(call GETENVOY_OUT_PATH,$(1),$(2)) endef $(foreach os,$(GOOSES),$(foreach arch,$(GOARCHS),$(eval $(call GEN_BIN_GOOS_GOARCH_TARGET,$(os),$(arch))))) diff --git a/ci/e2e/linux/run_tests.sh b/ci/e2e/linux/run_tests.sh index 4a0f983c..4a5fc7b9 100755 --- a/ci/e2e/linux/run_tests.sh +++ b/ci/e2e/linux/run_tests.sh @@ -38,5 +38,5 @@ docker-compose up -d # run e2e tests on a `getenvoy` binary built by the upstream job export E2E_GETENVOY_BINARY="${WORKSPACE_DIR}/build/bin/linux/amd64/getenvoy" -# run e2e tests with '-ginkgo.v' flag to be able to see the progress -${WORKSPACE_DIR}/build/bin/linux/amd64/e2e -ginkgo.v +# run e2e tests with '-v' flag to be able to see the progress +go test github.com/tetratelabs/getenvoy/test/e2e -test.parallel 1 -test.failfast -v ./test/e2e/... diff --git a/ci/e2e/macos/run_tests.sh b/ci/e2e/macos/run_tests.sh index 2ddc4e5b..a5166c13 100755 --- a/ci/e2e/macos/run_tests.sh +++ b/ci/e2e/macos/run_tests.sh @@ -41,4 +41,4 @@ docker-compose up -d export E2E_GETENVOY_BINARY="${WORKSPACE_DIR}/build/bin/darwin/amd64/getenvoy" # run e2e tests with '-ginkgo.v' flag to be able to see the progress -${WORKSPACE_DIR}/build/bin/darwin/amd64/e2e -ginkgo.v +go test github.com/tetratelabs/getenvoy/test/e2e -test.parallel 1 -test.failfast -v ./test/e2e/... diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go deleted file mode 100644 index 74d629d2..00000000 --- a/test/e2e/e2e_suite_test.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2020 Tetrate -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e_test - -import ( - "io/ioutil" - "os" - "path/filepath" - "testing" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - e2e "github.com/tetratelabs/getenvoy/test/e2e/util" -) - -func TestEndToEnd(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "e2e Suite") -} - -var _ = BeforeSuite(func() { - path, err := e2e.Env.GetEnvoyBinary() - Expect(err).NotTo(HaveOccurred()) - e2e.GetEnvoyBinaryPath = path -}) - -var ( - // GetEnvoy is a convenient alias. - GetEnvoy = e2e.GetEnvoy -) - -// tempDir represents a unique temporary directory made available for every test case. -var tempDir string - -var _ = BeforeEach(func() { - dir, err := ioutil.TempDir("", "") - Expect(err).NotTo(HaveOccurred()) - dir, err = filepath.EvalSymlinks(dir) - Expect(err).NotTo(HaveOccurred()) - tempDir = dir -}) - -var _ = AfterEach(func() { - if tempDir != "" { - Expect(os.RemoveAll(tempDir)).To(Succeed()) - } -}) - -func CleanUpExtensionDir(dir string) { - Expect(os.Chdir(dir)).To(Succeed()) - clean := GetEnvoy("extension clean") - _, _, err := clean.Exec() - Expect(err).To(Succeed()) -} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go new file mode 100644 index 00000000..9ada6f40 --- /dev/null +++ b/test/e2e/e2e_test.go @@ -0,0 +1,162 @@ +// Copyright 2021 Tetrate +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package e2e_test + +import ( + "io/ioutil" + "os" + "path/filepath" + "regexp" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tetratelabs/log" + + "github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension" + e2e "github.com/tetratelabs/getenvoy/test/e2e/util" +) + +var ( + // GetEnvoy is a convenient alias. + GetEnvoy = e2e.GetEnvoy +) + +// stripAnsiEscapeRegexp is a regular expression to clean ANSI Control sequences +// feat https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python#33925425 +var stripAnsiEscapeRegexp = regexp.MustCompile(`(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]`) + +func requireEnvoyBinaryPath(t *testing.T) { + path, err := e2e.Env.GetEnvoyBinary() + require.NoError(t, err, `error reading path to getenvoy binary`) + e2e.GetEnvoyBinaryPath = path +} + +// requireNewTempDir creates a new directory. The function returned cleans it up. +func requireNewTempDir(t *testing.T) (string, func()) { + d, err := ioutil.TempDir("", "") + if err != nil { + require.NoError(t, err, `ioutil.TempDir("", "") erred`) + } + dir := requireAbsDir(t, d) + return dir, func() { + e := os.RemoveAll(dir) + require.NoError(t, e, `error removing directory: %v`, dir) + } +} + +// RequireChDir will os.Chdir into the indicated dir, panicing on any problem. +// The function returned reverts to the original. +func requireChDir(t *testing.T, d string) func() { + // Save previous working directory to that it can be reverted later. + previous, err := os.Getwd() + require.NoError(t, err, `error determining current directory`) + + // Now, actually change to the directory. + err = os.Chdir(d) + require.NoError(t, err, `error changing to directory: %v`, d) + return func() { + e := os.Chdir(previous) + require.NoError(t, e, `error changing to directory: %v`, previous) + } +} + +// requireAbsDir runs filepath.Abs and ensures there are no errors and the input is a directory. +func requireAbsDir(t *testing.T, d string) string { + dir, err := filepath.Abs(d) + require.NoError(t, err, `error determining absolute directory: %v`, d) + require.DirExists(t, dir, `directory doesn't exist': %v`, dir) + dir, err = filepath.EvalSymlinks(dir) + require.NoError(t, err, `filepath.EvalSymlinks(%s) erred`, dir) + require.NotEmpty(t, dir, `filepath.EvalSymlinks(%s) returned ""`) + return dir +} + +// Command gives us an interface needed for testing GetEnvoy +type Command interface { + Exec() (string, string, error) +} + +// requireExecNoStdout invokes the command and returns its stderr if successful and stdout is empty. +func requireExecNoStdout(t *testing.T, cmd Command) string { + stdout, stderr := requireExec(t, cmd) + require.Empty(t, stdout, `expected no stdout running [%v]`, cmd) + require.NotEmpty(t, stderr, `expected stderr running [%v]`, cmd) + return stderr +} + +// requireExecNoStderr invokes the command and returns its stdout if successful and stderr is empty. +func requireExecNoStderr(t *testing.T, cmd Command) string { + stdout, stderr := requireExec(t, cmd) + require.NotEmpty(t, stdout, `expected stdout running [%v]`, cmd) + require.Empty(t, stderr, `expected no stderr running [%v]`, cmd) + return stdout +} + +// requireExec invokes the command and returns its (stdout, stderr) if successful. +func requireExec(t *testing.T, cmd Command) (string, string) { + log.Infof(`running [%v]`, cmd) + stdout, stderr, err := cmd.Exec() + + require.NoError(t, err, `error running [%v]`, cmd) + return stdout, stderr +} + +// requireExtensionInit is useful for tests that depend on "getenvoy extension init" as a prerequisite. +func requireExtensionInit(t *testing.T, workDir string, category extension.Category, language extension.Language, name string) { + cmd := GetEnvoy("extension init"). + Arg(workDir). + Arg("--category").Arg(string(category)). + Arg("--language").Arg(string(language)). + Arg("--name").Arg(name) + // stderr returned is not tested because doing so is redundant to TestGetEnvoyExtensionInit. + _ = requireExecNoStdout(t, cmd) +} + +// extensionWasmPath returns the language-specific location of the extension.wasm. +func extensionWasmPath(language extension.Language) string { + switch language { + case extension.LanguageRust: + return filepath.Join("target", "getenvoy", "extension.wasm") + case extension.LanguageTinyGo: + return filepath.Join("build", "extension.wasm") + } + panic("unsupported language " + language) +} + +// requireExtensionInit is useful for tests that depend on "getenvoy extension build" as a prerequisite. +// The result of calling this is the bytes representing the built wasm +func requireExtensionBuild(t *testing.T, language extension.Language, workDir string) []byte { + cmd := GetEnvoy("extension build").Args(e2e.Env.GetBuiltinContainerOptions()...) + // stderr returned is not tested because doing so is redundant to TestGetEnvoyExtensionInit. + _ = requireExecNoStderr(t, cmd) + + extensionWasmFile := filepath.Join(workDir, extensionWasmPath(language)) + require.FileExists(t, extensionWasmFile, `extension wasm file %s missing after running [%v]`, extensionWasmFile, cmd) + + wasmBytes, err := ioutil.ReadFile(extensionWasmFile) + require.NoError(t, err, `error reading %s after running [%v]: %s`, extensionWasmFile, cmd) + require.NotEmpty(t, wasmBytes, `%s empty after running [%v]`, extensionWasmFile, cmd) + return wasmBytes +} + +// requireExtensionClean is useful for tests that depend on "getenvoy extension clean" on completion. +// (stdout, stderr) returned are not tested because they can both be empty. +func requireExtensionClean(t *testing.T, workDir string) { + err := os.Chdir(workDir) + require.NoError(t, err, `error changing to directory: %v`, workDir) + + cmd := GetEnvoy("extension clean") + _, _ = requireExec(t, cmd) +} diff --git a/test/e2e/getenvoy_extension_build_test.go b/test/e2e/getenvoy_extension_build_test.go index f5385789..9a5a1f45 100644 --- a/test/e2e/getenvoy_extension_build_test.go +++ b/test/e2e/getenvoy_extension_build_test.go @@ -15,70 +15,45 @@ package e2e_test import ( - "os" "path/filepath" + "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" + "github.com/stretchr/testify/require" - workspaces "github.com/tetratelabs/getenvoy/pkg/extension/workspace" - toolchains "github.com/tetratelabs/getenvoy/pkg/extension/workspace/toolchain" e2e "github.com/tetratelabs/getenvoy/test/e2e/util" ) -var _ = Describe("getenvoy extension build", func() { - - type testCase e2e.CategoryLanguageTuple - - testCases := func() []TableEntry { - testCases := make([]TableEntry, 0) - for _, combination := range e2e.GetCategoryLanguageCombinations() { - testCases = append(testCases, Entry(combination.String(), testCase(combination))) - } - return testCases +// TestGetEnvoyExtensionBuild runs the equivalent of "getenvoy extension build" for a matrix of extension.Categories and +// extension.Languages. "getenvoy extension init" is a prerequisite, so run first. +// +// Note: "getenvoy extension build" can be extremely slow due to implicit responsibilities such as downloading modules +// or compilation. This uses Docker, so changes to the Dockerfile or contents like "commands.sh" effect performance. +func TestGetEnvoyExtensionBuild(t *testing.T) { + const extensionName = "getenvoy_extension_build" + requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy + + for _, test := range e2e.GetCategoryLanguageCombinations() { + test := test // pin! see https://github.com/kyoh86/scopelint for why + + t.Run(test.String(), func(t *testing.T) { + workDir, removeWorkDir := requireNewTempDir(t) + defer removeWorkDir() + + revertChDir := requireChDir(t, workDir) + defer revertChDir() + + // test requires "get envoy extension init" to have succeeded + requireExtensionInit(t, workDir, test.Category, test.Language, extensionName) + defer requireExtensionClean(t, workDir) + + // "getenvoy extension build" only returns stdout because `docker run -t` redirects stderr to stdout. + // We don't verify stdout because it is low signal vs looking at files created. + cmd := GetEnvoy("extension build").Args(e2e.Env.GetBuiltinContainerOptions()...) + _ = requireExecNoStderr(t, cmd) + + // Verify the extension built + extensionWasmFile := filepath.Join(workDir, extensionWasmPath(test.Language)) + require.FileExists(t, extensionWasmFile, `extension wasm file %s missing after running [%v]`, extensionWasmFile, cmd) + }) } - - const extensionName = "my.extension" - - DescribeTable("should build a *.wasm file", - func(given testCase) { - By("choosing the output directory") - outputDir := filepath.Join(tempDir, "new") - defer CleanUpExtensionDir(outputDir) - - By("running `extension init` command") - _, _, err := GetEnvoy("extension init"). - Arg(outputDir). - Arg("--category").Arg(given.Category.String()). - Arg("--language").Arg(given.Language.String()). - Arg("--name").Arg(extensionName). - Exec() - Expect(err).NotTo(HaveOccurred()) - - By("changing to the output directory") - err = os.Chdir(outputDir) - Expect(err).NotTo(HaveOccurred()) - - By("running `extension build` command") - stdout, stderr, err := GetEnvoy("extension build"). - Args(e2e.Env.GetBuiltinContainerOptions()...). - Exec() - Expect(err).NotTo(HaveOccurred()) - - By("verifying stdout/stderr") - // apparently, use of `-t` option in `docker run` causes stderr to be incorporated into stdout - Expect(stdout).NotTo(BeEmpty()) - Expect(stderr).To(BeEmpty()) - - By("verifying *.wasm file") - workspace, err := workspaces.GetWorkspaceAt(outputDir) - Expect(err).NotTo(HaveOccurred()) - Expect(workspace).NotTo(BeNil()) - toolchain, err := toolchains.LoadToolchain(toolchains.Default, workspace) - Expect(err).NotTo(HaveOccurred()) - Expect(toolchain).NotTo(BeNil()) - }, - testCases()..., - ) -}) +} diff --git a/test/e2e/getenvoy_extension_examples_test.go b/test/e2e/getenvoy_extension_examples_test.go index e32a9f5b..42fa08b1 100644 --- a/test/e2e/getenvoy_extension_examples_test.go +++ b/test/e2e/getenvoy_extension_examples_test.go @@ -16,124 +16,97 @@ package e2e_test import ( "fmt" - "os" "path/filepath" + "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" + "github.com/stretchr/testify/require" "github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension" e2e "github.com/tetratelabs/getenvoy/test/e2e/util" ) -var _ = Describe("getenvoy extension examples", func() { +// TestGetEnvoyExtensionExampleAdd runs the equivalent of "getenvoy extension example XXX" commands for a matrix of +// extension.Categories and extension.Languages. +// +// "getenvoy extension example XXX" should be fast and reliable because they don't use Docker. +func TestGetEnvoyExtensionExample(t *testing.T) { + const extensionName = "getenvoy_extension_example" + requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy - type testCase e2e.CategoryLanguageTuple + for _, test := range e2e.GetCategoryLanguageCombinations() { + test := test // pin! see https://github.com/kyoh86/scopelint for why - testCases := func() []TableEntry { - testCases := make([]TableEntry, 0) - for _, combination := range e2e.GetCategoryLanguageCombinations() { - testCases = append(testCases, Entry(combination.String(), testCase(combination))) - } - return testCases - } + t.Run(test.String(), func(t *testing.T) { + var extensionConfigFileName string + switch test.Language { + case extension.LanguageTinyGo: + extensionConfigFileName = "extension.txt" + default: + extensionConfigFileName = "extension.json" + } - const extensionName = "my.extension" + workDir, removeWorkDir := requireNewTempDir(t) + defer removeWorkDir() - DescribeTable("should create extension in a new directory", - func(given testCase) { - By("choosing the output directory") - outputDir := filepath.Join(tempDir, "new") - defer CleanUpExtensionDir(outputDir) + revertChDir := requireChDir(t, workDir) + defer revertChDir() - By("running `extension init` command") - _, _, err := GetEnvoy("extension init"). - Arg(outputDir). - Arg("--category").Arg(given.Category.String()). - Arg("--language").Arg(given.Language.String()). - Arg("--name").Arg(extensionName). - Exec() - Expect(err).NotTo(HaveOccurred()) + // "getenvoy extension example XXX" commands require an extension init to succeed + requireExtensionInit(t, workDir, test.Category, test.Language, extensionName) + defer requireExtensionClean(t, workDir) - By("changing to the output directory") - err = os.Chdir(outputDir) - Expect(err).NotTo(HaveOccurred()) + // "getenvoy extension examples list" should start empty + cmd := GetEnvoy("extension examples list") + stderr := requireExecNoStdout(t, cmd) + require.Equal(t, `Extension has no example setups. - By("running `extension examples list` command") - stdout, stderr, err := GetEnvoy("extension examples list").Exec() - Expect(err).NotTo(HaveOccurred()) +Use "getenvoy extension examples add --help" for more information on how to add one. +`, stderr, `invalid stderr running [%v]`, cmd) - By("verifying stdout/stderr") - Expect(stdout).To(Equal(``)) - Expect(stderr).To(Equal(`Extension has no example setups. + // "getenvoy extension examples add" should result in stderr describing files created. + cmd = GetEnvoy("extension examples add") + stderr = requireExecNoStdout(t, cmd) -Use "getenvoy extension examples add --help" for more information on how to add one. -`)) + exampleFiles := []string{ + filepath.Join(workDir, ".getenvoy/extension/examples/default/README.md"), + filepath.Join(workDir, ".getenvoy/extension/examples/default/envoy.tmpl.yaml"), + filepath.Join(workDir, ".getenvoy/extension/examples/default/example.yaml"), + fmt.Sprintf(".getenvoy/extension/examples/default/%s", extensionConfigFileName), + } - By("running `extension examples add` command") - stdout, stderr, err = GetEnvoy("extension examples add").Exec() - Expect(err).NotTo(HaveOccurred()) + exampleFileText := fmt.Sprintf(` +* .getenvoy/extension/examples/default/README.md +* .getenvoy/extension/examples/default/envoy.tmpl.yaml +* .getenvoy/extension/examples/default/example.yaml +* .getenvoy/extension/examples/default/%s +`, extensionConfigFileName) - By("verifying stdout/stderr") - var extensionConfigFileName string - switch given.Language { - case extension.LanguageTinyGo: - extensionConfigFileName = "extension.txt" - default: - extensionConfigFileName = "extension.json" + // Check stderr mentions the files created + require.Equal(t, fmt.Sprintf("Scaffolding a new example setup:%sDone!\n", exampleFileText), + stderr, `invalid stderr running [%v]`, cmd) + + // Check the files mentioned actually exist + for _, path := range exampleFiles { + require.FileExists(t, path, `example file %s missing after running [%v]`, path, cmd) } - Expect(stdout).To(Equal(``)) - Expect(stderr).To(MatchRegexp(`^\QScaffolding a new example setup:\E\n`)) - Expect(stderr).To(MatchRegexp(`\Q* .getenvoy/extension/examples/default/README.md\E\n`)) - Expect(stderr).To(MatchRegexp(`\Q* .getenvoy/extension/examples/default/envoy.tmpl.yaml\E\n`)) - Expect(stderr).To(MatchRegexp(`\Q* .getenvoy/extension/examples/default/example.yaml\E\n`)) - Expect(stderr).To(MatchRegexp( - fmt.Sprintf(`\Q* .getenvoy/extension/examples/default/%s\E\n`, extensionConfigFileName))) - Expect(stderr).To(MatchRegexp(`\QDone!\E\n$`)) - - By("verifying output directory") - Expect(filepath.Join(outputDir, ".getenvoy/extension/examples/default/README.md")).To(BeAnExistingFile()) - Expect(filepath.Join(outputDir, ".getenvoy/extension/examples/default/envoy.tmpl.yaml")).To(BeAnExistingFile()) - Expect(filepath.Join(outputDir, ".getenvoy/extension/examples/default/example.yaml")).To(BeAnExistingFile()) - Expect(filepath.Join(outputDir, - fmt.Sprintf(".getenvoy/extension/examples/default/%s", extensionConfigFileName))).To(BeAnExistingFile()) - - By("running `extension examples list` command") - stdout, stderr, err = GetEnvoy("extension examples list").Exec() - Expect(err).NotTo(HaveOccurred()) - - By("verifying stdout/stderr") - Expect(stdout).To(Equal(`EXAMPLE -default -`)) - Expect(stderr).To(BeEmpty()) - - By("running `extension examples remove` command") - stdout, stderr, err = GetEnvoy("extension examples remove --name default").Exec() - Expect(err).NotTo(HaveOccurred()) - - By("verifying stdout/stderr") - Expect(stdout).To(Equal(``)) - Expect(stderr).To(MatchRegexp(`^\QRemoving example setup:\E\n`)) - Expect(stderr).To(MatchRegexp(`\Q* .getenvoy/extension/examples/default/README.md\E\n`)) - Expect(stderr).To(MatchRegexp(`\Q* .getenvoy/extension/examples/default/envoy.tmpl.yaml\E\n`)) - Expect(stderr).To(MatchRegexp(`\Q* .getenvoy/extension/examples/default/example.yaml\E\n`)) - Expect(stderr).To(MatchRegexp(fmt.Sprintf( - `\Q* .getenvoy/extension/examples/default/%s\E\n`, extensionConfigFileName))) - Expect(stderr).To(MatchRegexp(`\QDone!\E\n$`)) - - By("running `extension examples list` command") - stdout, stderr, err = GetEnvoy("extension examples list").Exec() - Expect(err).NotTo(HaveOccurred()) - - By("verifying stdout/stderr") - Expect(stdout).To(Equal(``)) - Expect(stderr).To(Equal(`Extension has no example setups. -Use "getenvoy extension examples add --help" for more information on how to add one. -`)) - }, - testCases()..., - ) -}) + // "getenvoy extension examples list" should now include an example + cmd = GetEnvoy("extension examples list") + stdout := requireExecNoStderr(t, cmd) + require.Equal(t, "EXAMPLE\ndefault\n", stdout, `invalid stdout running [%v]`, cmd) + + // "getenvoy extension examples add" should result in stderr describing files created. + cmd = GetEnvoy("extension examples remove --name default") + stderr = requireExecNoStdout(t, cmd) + + // Check stderr mentions the files removed + require.Equal(t, fmt.Sprintf("Removing example setup:%sDone!\n", exampleFileText), + stderr, `invalid stderr running [%v]`, cmd) + + // Check the files mentioned actually were removed + for _, path := range exampleFiles { + require.NoFileExists(t, path, `example file %s still exists after running [%v]`, path, cmd) + } + }) + } +} diff --git a/test/e2e/getenvoy_extension_init_test.go b/test/e2e/getenvoy_extension_init_test.go index 0dd4e0f7..f6bd5d42 100644 --- a/test/e2e/getenvoy_extension_init_test.go +++ b/test/e2e/getenvoy_extension_init_test.go @@ -15,94 +15,89 @@ package e2e_test import ( - "os" + "fmt" "path/filepath" + "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" + "github.com/stretchr/testify/require" workspaces "github.com/tetratelabs/getenvoy/pkg/extension/workspace" + "github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension" + toolchains "github.com/tetratelabs/getenvoy/pkg/extension/workspace/toolchain" e2e "github.com/tetratelabs/getenvoy/test/e2e/util" ) -var _ = Describe("getenvoy extension init", func() { - - type testCase e2e.CategoryLanguageTuple - - testCases := func() []TableEntry { - testCases := make([]TableEntry, 0) - for _, combination := range e2e.GetCategoryLanguageCombinations() { - testCases = append(testCases, Entry(combination.String(), testCase(combination))) - } - return testCases +// TestGetEnvoyExtensionInit runs the equivalent of "getenvoy extension init" for a matrix of extension.Categories and +// extension.Languages. +// +// "getenvoy extension init" should be fast and reliable because it doesn't use Docker. +func TestGetEnvoyExtensionInit(t *testing.T) { + const extensionName = "getenvoy_extension_init" + requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy + + type testTuple struct { + testName string + extension.Category + extension.Language + currentDirectory bool } - const extensionName = "my.extension" - - VerifyStdoutStderr := func(stdout string, stderr string, outputDir string) { - Expect(stdout).To(Equal(``)) - Expect(stderr).To(MatchRegexp(`^\QScaffolding a new extension:\E\n`)) - Expect(stderr).To(MatchRegexp(`\QGenerating files in %s:\E\n`, outputDir)) - Expect(stderr).To(MatchRegexp(`\Q* .getenvoy/extension/extension.yaml\E\n`)) - Expect(stderr).To(MatchRegexp(`\QDone!\E\n$`)) + tests := make([]testTuple, 0) + for _, c := range e2e.GetCategoryLanguageCombinations() { + tests = append(tests, + testTuple{c.String() + "-currentDirectory", c.Category, c.Language, true}, + testTuple{c.String() + "-newDirectory", c.Category, c.Language, false}, + ) } - VerifyOutputDir := func(given testCase, outputDir string) { - workspace, err := workspaces.GetWorkspaceAt(outputDir) - Expect(err).NotTo(HaveOccurred()) - Expect(workspace.GetExtensionDescriptor().Name).To(Equal(extensionName)) - Expect(workspace.GetExtensionDescriptor().Category).To(Equal(given.Category)) - Expect(workspace.GetExtensionDescriptor().Language).To(Equal(given.Language)) + for _, test := range tests { + test := test // pin! see https://github.com/kyoh86/scopelint for why + + t.Run(test.testName, func(t *testing.T) { + workDir, removeWorkDir := requireNewTempDir(t) + defer removeWorkDir() + + revertChDir := requireChDir(t, workDir) + defer revertChDir() + + if !test.currentDirectory { + workDir = filepath.Join(workDir, "newDirectory") + } + + // "getenvoy extension init" should result in stderr describing files created. + cmd := GetEnvoy("extension init"). + Arg(workDir). + Arg("--category").Arg(test.Category.String()). + Arg("--language").Arg(test.Language.String()). + Arg("--name").Arg(extensionName) + stderr := requireExecNoStdout(t, cmd) + + // Check that the contents look valid for the inputs. + for _, regex := range []string{ + `^\QScaffolding a new extension:\E\n`, + fmt.Sprintf(`\QGenerating files in %s:\E\n`, workDir), + `\Q* .getenvoy/extension/extension.yaml\E\n`, + `\QDone!\E\n$`, + } { + require.Regexp(t, regex, stderr, `invalid stderr running [%v]`, cmd) + } + + // Check to see that the extension.yaml mentioned in stderr exists. + // Note: we don't check all files as extensions are language-specific. + require.FileExists(t, filepath.Join(workDir, ".getenvoy/extension/extension.yaml"), `extension.yaml missing after running [%v]`, cmd) + + // Check the generated extension.yaml includes values we passed and includes the default toolchain. + workspace, err := workspaces.GetWorkspaceAt(workDir) + require.NoError(t, err, `error getting workspace after running [%v]`, cmd) + require.NotNil(t, workspace, `nil workspace running [%v]`, cmd) + require.Equal(t, extensionName, workspace.GetExtensionDescriptor().Name, `wrong extension name running [%v]`, cmd) + require.Equal(t, test.Category, workspace.GetExtensionDescriptor().Category, `wrong extension category running [%v]`, cmd) + require.Equal(t, test.Language, workspace.GetExtensionDescriptor().Language, `wrong extension language running [%v]`, cmd) + + // Check the default toolchain is loadable + toolchain, err := toolchains.LoadToolchain(toolchains.Default, workspace) + require.NoError(t, err, `error loading toolchain running [%v]`, cmd) + require.NotNil(t, toolchain, `nil toolchain running [%v]`, cmd) + }) } - - DescribeTable("should create extension in a new directory", - func(given testCase) { - By("choosing the output directory") - outputDir := filepath.Join(tempDir, "new") - - By("running `extension init` command") - stdout, stderr, err := GetEnvoy("extension init"). - Arg(outputDir). - Arg("--category").Arg(given.Category.String()). - Arg("--language").Arg(given.Language.String()). - Arg("--name").Arg(extensionName). - Exec() - Expect(err).NotTo(HaveOccurred()) - - By("verifying stdout/stderr") - VerifyStdoutStderr(stdout, stderr, outputDir) - - By("verifying output directory") - VerifyOutputDir(given, outputDir) - }, - testCases()..., - ) - - DescribeTable("should create extension in the current directory", - func(given testCase) { - By("choosing the output directory") - outputDir := tempDir - defer CleanUpExtensionDir(outputDir) - - By("changing to the output directory") - err := os.Chdir(outputDir) - Expect(err).NotTo(HaveOccurred()) - - By("running `extension init` command") - stdout, stderr, err := GetEnvoy("extension init"). - Arg("--category").Arg(given.Category.String()). - Arg("--language").Arg(given.Language.String()). - Arg("--name").Arg(extensionName). - Exec() - Expect(err).NotTo(HaveOccurred()) - - By("verifying stdout/stderr") - VerifyStdoutStderr(stdout, stderr, outputDir) - - By("verifying output directory") - VerifyOutputDir(given, outputDir) - }, - testCases()..., - ) -}) +} diff --git a/test/e2e/getenvoy_extension_push_test.go b/test/e2e/getenvoy_extension_push_test.go index a39ee30a..d553525f 100644 --- a/test/e2e/getenvoy_extension_push_test.go +++ b/test/e2e/getenvoy_extension_push_test.go @@ -17,205 +17,90 @@ package e2e_test import ( "fmt" "io/ioutil" - "os" "path/filepath" - "regexp" - "sync" - "time" + "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" + "github.com/stretchr/testify/require" - "github.com/tetratelabs/getenvoy/pkg/common" "github.com/tetratelabs/getenvoy/pkg/extension/wasmimage" - workspaces "github.com/tetratelabs/getenvoy/pkg/extension/workspace" - toolchains "github.com/tetratelabs/getenvoy/pkg/extension/workspace/toolchain" - e2e "github.com/tetratelabs/getenvoy/test/e2e/util" - utilenvoy "github.com/tetratelabs/getenvoy/test/e2e/util/envoy" + "github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension" ) -const ( - localRegistryWasmImageRef = "localhost:5000/getenvoy/sample:latest" -) - -var _ = Describe("getenvoy extension push", func() { - var debugDir string - - BeforeEach(func() { - debugDir = filepath.Join(common.DefaultHomeDir(), "debug") - }) - - var backupDebugDir string - - BeforeEach(func() { - _, err := ioutil.ReadDir(debugDir) - if os.IsNotExist(err) { - return - } - Expect(err).NotTo(HaveOccurred()) - - By("backing up GetEnvoy debug dir") - backupDir, err := ioutil.TempDir(filepath.Dir(debugDir), "debug") - Expect(err).NotTo(HaveOccurred()) - err = os.RemoveAll(backupDir) - Expect(err).NotTo(HaveOccurred()) - - err = os.Rename(debugDir, backupDir) - Expect(err).NotTo(HaveOccurred()) - backupDebugDir = backupDir - }) - - AfterEach(func() { - if backupDebugDir == "" { - return - } - By("restoring GetEnvoy debug dir from backup") - err := os.RemoveAll(debugDir) - Expect(err).NotTo(HaveOccurred()) - err = os.Rename(backupDebugDir, debugDir) - Expect(err).NotTo(HaveOccurred()) - }) - - type testCase e2e.CategoryLanguageTuple - - testCases := func() []TableEntry { - testCases := make([]TableEntry, 0) - for _, combination := range e2e.GetCategoryLanguageCombinations() { - testCases = append(testCases, Entry(combination.String(), testCase(combination))) - } - return testCases +// TestGetEnvoyExtensionPush runs the equivalent of "getenvoy extension push". "getenvoy extension init" and +// "getenvoy extension build" are a prerequisites, so run first. +// +// This test does not attempt to use the image built as that would be redundant to other tests. Rather, this focuses on +// whether we can read back exactly what was pushed to the registry. +// +// Note: "getenvoy extension build" can be extremely slow due to implicit responsibilities such as downloading modules +// or compilation. This uses Docker, so changes to the Dockerfile or contents like "commands.sh" effect performance. +func TestGetEnvoyExtensionPush(t *testing.T) { + const extensionName = "getenvoy_extension_push" + // localRegistryWasmImageRef corresponds to a Docker container running the image "registry:2" + const localRegistryWasmImageRef = "localhost:5000/getenvoy/" + extensionName + // When unspecified, we default the tag to Docker's default "latest". Note: recent tools enforce qualifying this! + const defaultTag = "latest" + requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy + + type testTuple struct { + name string + extension.Category + extension.Language } - AtMostOnce := func(fn func()) func() { - var once sync.Once - return func() { - once.Do(fn) - } + // Push is not language-specific, so we don't need to test a large matrix, and doing so would slow down e2e runtime. + // Instead, we choose something that executes "getenvoy extension build" quickly. + tests := []testTuple{ + {"tinygo HTTP filter", extension.EnvoyHTTPFilter, extension.LanguageTinyGo}, } - // TODO(musaprg): write teardown process for local registries if it's needed - - const extensionName = "my.extension" - - const terminateTimeout = 2 * time.Minute - - DescribeTable("should push a *.wasm file", - func(given testCase) { - By("choosing the output directory") - outputDir := filepath.Join(tempDir, "new") - defer CleanUpExtensionDir(outputDir) - - By("running `extension init` command") - _, _, err := GetEnvoy("extension init"). - Arg(outputDir). - Arg("--category").Arg(given.Category.String()). - Arg("--language").Arg(given.Language.String()). - Arg("--name").Arg(extensionName). - Exec() - Expect(err).NotTo(HaveOccurred()) - - By("changing to the output directory") - err = os.Chdir(outputDir) - Expect(err).NotTo(HaveOccurred()) - - By("running `extension build` command") - stdout, stderr, err := GetEnvoy("extension build"). - Args(e2e.Env.GetBuiltinContainerOptions()...). - Exec() - Expect(err).NotTo(HaveOccurred()) - - By("verifying stdout/stderr") - // apparently, use of `-t` option in `docker run` causes stderr to be incorporated into stdout - Expect(stdout).NotTo(BeEmpty()) - Expect(stderr).To(BeEmpty()) - - By("verifying *.wasm file") - workspace, err := workspaces.GetWorkspaceAt(outputDir) - Expect(err).NotTo(HaveOccurred()) - Expect(workspace).NotTo(BeNil()) - toolchain, err := toolchains.LoadToolchain(toolchains.Default, workspace) - Expect(err).NotTo(HaveOccurred()) - Expect(toolchain).NotTo(BeNil()) - - By("running `extension push` command") - _, _, err = GetEnvoy("extension push").Arg(localRegistryWasmImageRef).Exec() - Expect(err).NotTo(HaveOccurred()) - Expect(stdout).NotTo(BeEmpty()) - Expect(stderr).To(BeEmpty()) - - By("pulling pushed wasm binary") + for _, test := range tests { + test := test // pin! see https://github.com/kyoh86/scopelint for why + + t.Run(test.name, func(t *testing.T) { + workDir, removeWorkDir := requireNewTempDir(t) + defer removeWorkDir() + + revertChDir := requireChDir(t, workDir) + defer revertChDir() + + // push requires "get envoy extension init" and "get envoy extension build" to have succeeded + requireExtensionInit(t, workDir, test.Category, test.Language, extensionName) + defer requireExtensionClean(t, workDir) + wasmBytes := requireExtensionBuild(t, test.Language, workDir) + + // After pushing, stderr should include the registry URL and the image tag. + cmd := GetEnvoy("extension push").Arg(localRegistryWasmImageRef) + stderr := requireExecNoStdout(t, cmd) + + // Assemble a fully-qualified image ref as we'll pull this later + imageRef := localRegistryWasmImageRef + ":" + defaultTag + + // Verify stderr shows the latest tag and the correct image ref + require.Contains(t, stderr, fmt.Sprintf(`Using default tag: %s +Pushed %s +digest: sha256`, defaultTag, imageRef), `unexpected stderr after running [%v]`, cmd) + + // Get a puller we can use to pull what we just pushed. puller, err := wasmimage.NewPuller(false, false) - Expect(err).NotTo(HaveOccurred()) - Expect(puller).NotTo(BeNil()) - dstPath := filepath.Join(outputDir, "pulled_extension.wasm") - _, err = puller.Pull(localRegistryWasmImageRef, dstPath) - Expect(err).NotTo(HaveOccurred()) - - By("running `extension run` command with pulled image") - _, se, cancel, errs := GetEnvoy("extension run"). - Args(e2e.Env.GetBuiltinContainerOptions()...). - Arg("--extension-file").Arg(dstPath). - Start() - - cancelCh := make(chan struct{}) - cancelGracefully := AtMostOnce(func() { - close(cancelCh) - - Expect(cancel()).To(Succeed()) - select { - case e := <-errs: - Expect(e).NotTo(HaveOccurred()) - case <-time.After(terminateTimeout): - Fail(fmt.Sprintf("getenvoy command didn't exit gracefully within %s", terminateTimeout)) - } - }) - // make sure to stop Envoy if test fails - defer cancelGracefully() - - // fail the test if `getenvoy extension run` exits with an error or unexpectedly - go func() { - select { - case e := <-errs: - Expect(e).NotTo(HaveOccurred(), "getenvoy command exited unexpectedly") - case <-cancelCh: - } - }() - - stderrLines := e2e.StreamLines(se).Named("stderr") - - By("waiting for Envoy Admin address to get logged") - adminAddressPattern := regexp.MustCompile(`admin address: ([^:]+:[0-9]+)`) - line, err := stderrLines.FirstMatch(adminAddressPattern).Wait(10 * time.Minute) // give time to compile the extension - Expect(err).NotTo(HaveOccurred()) - adminAddress := adminAddressPattern.FindStringSubmatch(line)[1] - - By("waiting for Envoy start-up to complete") - stderrLines.FirstMatch(regexp.MustCompile(`starting main dispatch loop`)).Wait(1 * time.Minute) - - By("verifying Envoy is ready") - envoyClient, err := utilenvoy.NewClient(adminAddress) - Expect(err).NotTo(HaveOccurred()) - Eventually(func() bool { - ready, e := envoyClient.IsReady() - return e == nil && ready - }, "60s", "100ms").Should(BeTrue()) - - By("verifying Wasm extensions have been created") - Eventually(func() bool { - stats, e := envoyClient.GetStats() - if e != nil { - return false - } - // at the moment, the only available Wasm metric is the number of Wasm VMs - concurrency := stats.GetMetric("server.concurrency") - activeWasmVms := stats.GetMetric("wasm.envoy.wasm.runtime.v8.active") - return concurrency != nil && activeWasmVms != nil && activeWasmVms.Value == concurrency.Value+2 - }, "60s", "100ms").Should(BeTrue()) - - By("signaling Envoy to stop") - cancelGracefully() - }, - testCases()..., - ) -}) + require.NoError(t, err, `error getting puller instance after running [%v]`, cmd) + require.NotNil(t, puller, `nil puller instance after running [%v]`, cmd) + + // Pull the wasm we just pushed, writing it to a local file. + dstPath := filepath.Join(workDir, "pulled_extension.wasm") + desc, err := puller.Pull(imageRef, dstPath) + require.NoError(t, err, `error pulling wasm after running [%v]: %s`, cmd) + + // Verify the pulled image descriptor is valid and the image file exists/ + require.Equal(t, "application/vnd.module.wasm.content.layer.v1+wasm", desc.MediaType, `invalid media type after running [%v]`, cmd) + require.Equal(t, "extension.wasm", desc.Annotations["org.opencontainers.image.title"], `invalid image title after running [%v]`, cmd) + require.FileExists(t, dstPath, `image not written after running [%v]`, cmd) + + // Verify the bytes pulled are exactly the same as what we pushed. + pulledBytes, err := ioutil.ReadFile(dstPath) + require.NoError(t, err, `error reading file wasm %s after running [%v]`, dstPath, cmd) + require.NotEmpty(t, wasmBytes, `%s empty after running [%v]`, dstPath, cmd) + require.Equal(t, wasmBytes, pulledBytes, `pulled bytes don't match source after running [%v]`, cmd) + }) + } +} diff --git a/test/e2e/getenvoy_extension_run_test.go b/test/e2e/getenvoy_extension_run_test.go index 8bb7c47f..b5a4b98d 100644 --- a/test/e2e/getenvoy_extension_run_test.go +++ b/test/e2e/getenvoy_extension_run_test.go @@ -15,154 +15,78 @@ package e2e_test import ( - "fmt" "io/ioutil" - "os" "path/filepath" "regexp" - "sync" + "testing" "time" "github.com/mholt/archiver" - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" + "github.com/stretchr/testify/require" + "github.com/tetratelabs/log" - "github.com/tetratelabs/getenvoy/pkg/common" e2e "github.com/tetratelabs/getenvoy/test/e2e/util" utilenvoy "github.com/tetratelabs/getenvoy/test/e2e/util/envoy" ) -var _ = Describe("getenvoy extension run", func() { - - var debugDir string - - BeforeEach(func() { - debugDir = filepath.Join(common.DefaultHomeDir(), "debug") - }) - - var backupDebugDir string - - BeforeEach(func() { - _, err := ioutil.ReadDir(debugDir) - if os.IsNotExist(err) { - return - } - Expect(err).NotTo(HaveOccurred()) - - By("backing up GetEnvoy debug dir") - backupDir, err := ioutil.TempDir(filepath.Dir(debugDir), "debug") - Expect(err).NotTo(HaveOccurred()) - err = os.RemoveAll(backupDir) - Expect(err).NotTo(HaveOccurred()) - - err = os.Rename(debugDir, backupDir) - Expect(err).NotTo(HaveOccurred()) - backupDebugDir = backupDir - }) - - AfterEach(func() { - if backupDebugDir == "" { - return - } - By("restoring GetEnvoy debug dir from backup") - err := os.RemoveAll(debugDir) - Expect(err).NotTo(HaveOccurred()) - err = os.Rename(backupDebugDir, debugDir) - Expect(err).NotTo(HaveOccurred()) - }) - - type testCase e2e.CategoryLanguageTuple - - testCases := func() []TableEntry { - testCases := make([]TableEntry, 0) - for _, combination := range e2e.GetCategoryLanguageCombinations() { - testCases = append(testCases, Entry(combination.String(), testCase(combination))) - } - return testCases - } +// TestGetEnvoyExtensionRun runs the equivalent of "getenvoy extension run" for a matrix of extension.Categories and +// extension.Languages. "getenvoy extension init" is a prerequisite, so run first. +// +// Note: "getenvoy extension run" can be extremely slow due to implicit responsibilities such as downloading modules +// or compilation. This uses Docker, so changes to the Dockerfile or contents like "commands.sh" effect performance. +func TestGetEnvoyExtensionRun(t *testing.T) { + const extensionName = "getenvoy_extension_run" + const terminateTimeout = 2 * time.Minute + requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy - AtMostOnce := func(fn func()) func() { - var once sync.Once - return func() { - once.Do(fn) - } - } + for _, test := range e2e.GetCategoryLanguageCombinations() { + test := test // pin! see https://github.com/kyoh86/scopelint for why - const extensionName = "my.extension" + t.Run(test.String(), func(t *testing.T) { + // We override the home directory ~/.getenvoy so as to not overwrite any debug info there. We need to do + // this to ensure presence of debug files are from this run, not another one. + homeDir, removeHomeDir := requireNewTempDir(t) + defer removeHomeDir() - const terminateTimeout = 2 * time.Minute + workDir, removeWorkDir := requireNewTempDir(t) + defer removeWorkDir() - DescribeTable("should create and run default example setup", - func(given testCase) { - By("choosing the output directory") - outputDir := filepath.Join(tempDir, "extension") - defer CleanUpExtensionDir(outputDir) - - By("running `extension init` command") - _, _, err := GetEnvoy("extension init"). - Arg(outputDir). - Arg("--category").Arg(given.Category.String()). - Arg("--language").Arg(given.Language.String()). - Arg("--name").Arg(extensionName). - Exec() - Expect(err).NotTo(HaveOccurred()) - - By("changing to the output directory") - err = os.Chdir(outputDir) - Expect(err).NotTo(HaveOccurred()) - - By("running `extension run` command") - _, stderr, cancel, errs := GetEnvoy("extension run --envoy-options '-l trace'"). - Args(e2e.Env.GetBuiltinContainerOptions()...). - Start() - - cancelCh := make(chan struct{}) - cancelGracefully := AtMostOnce(func() { - close(cancelCh) - - Expect(cancel()).To(Succeed()) - select { - case e := <-errs: - Expect(e).NotTo(HaveOccurred()) - case <-time.After(terminateTimeout): - Fail(fmt.Sprintf("getenvoy command didn't exit gracefully within %s", terminateTimeout)) - } - }) - // make sure to stop Envoy if test fails - defer cancelGracefully() - - // fail the test if `getenvoy extension run` exits with an error or unexpectedly - go func() { - defer GinkgoRecover() - select { - case e := <-errs: - Expect(e).NotTo(HaveOccurred(), "getenvoy command exited unexpectedly") - case <-cancelCh: - } - }() + revertChDir := requireChDir(t, workDir) + defer revertChDir() + + // run requires "get envoy extension init" to have succeeded + requireExtensionInit(t, workDir, test.Category, test.Language, extensionName) + defer requireExtensionClean(t, workDir) + + // "getenvoy extension run" only returns stdout because `docker run -t` redirects stderr to stdout. + cmd := GetEnvoy("extension run --envoy-options '-l trace'"). + Args("--home-dir", homeDir). + Args(e2e.Env.GetBuiltinContainerOptions()...) + _, stderr, terminate := cmd.Start(t, terminateTimeout) + defer terminate() stderrLines := e2e.StreamLines(stderr).Named("stderr") - By("waiting for Envoy Admin address to get logged") + log.Infof(`waiting for Envoy Admin address to get logged after running [%v]`, cmd) adminAddressPattern := regexp.MustCompile(`admin address: ([^:]+:[0-9]+)`) line, err := stderrLines.FirstMatch(adminAddressPattern).Wait(10 * time.Minute) // give time to compile the extension - Expect(err).NotTo(HaveOccurred()) + require.NoError(t, err, `error parsing admin address from stderr of [%v]`, cmd) adminAddress := adminAddressPattern.FindStringSubmatch(line)[1] - By("waiting for Envoy start-up to complete") - stderrLines.FirstMatch(regexp.MustCompile(`starting main dispatch loop`)).Wait(1 * time.Minute) + log.Infof(`waiting for Envoy start-up to complete after running [%v]`, cmd) + _, err = stderrLines.FirstMatch(regexp.MustCompile(`starting main dispatch loop`)).Wait(1 * time.Minute) + require.NoError(t, err, `error parsing startup from stderr of [%v]`, cmd) - By("verifying Envoy is ready") + log.Infof(`waiting for Envoy client to connect after running [%v]`, cmd) envoyClient, err := utilenvoy.NewClient(adminAddress) - Expect(err).NotTo(HaveOccurred()) - Eventually(func() bool { + require.NoError(t, err, `error from envoy client %s after running [%v]`, adminAddress, cmd) + require.Eventually(t, func() bool { ready, e := envoyClient.IsReady() return e == nil && ready - }, "60s", "100ms").Should(BeTrue()) + }, 1*time.Minute, 100*time.Millisecond, `envoy client %s never ready after running [%v]`, adminAddress, cmd) - By("verifying Wasm extensions have been created") - Eventually(func() bool { + log.Infof(`waiting for Wasm extensions after running [%v]`, cmd) + require.Eventually(t, func() bool { stats, e := envoyClient.GetStats() if e != nil { return false @@ -171,25 +95,30 @@ var _ = Describe("getenvoy extension run", func() { concurrency := stats.GetMetric("server.concurrency") activeWasmVms := stats.GetMetric("wasm.envoy.wasm.runtime.v8.active") return concurrency != nil && activeWasmVms != nil && activeWasmVms.Value == concurrency.Value+2 - }, "60s", "100ms").Should(BeTrue()) + }, 1*time.Minute, 100*time.Millisecond, `wasm stats never found after running [%v]`, adminAddress, cmd) - By("signaling Envoy to stop") - cancelGracefully() + log.Infof(`stopping Envoy after running [%v]`, cmd) + terminate() - By("verifying the debug dump of Envoy state has been taken") + // verify the debug dump of Envoy state has been taken + debugDir := filepath.Join(homeDir, "debug") files, err := ioutil.ReadDir(debugDir) - Expect(err).NotTo(HaveOccurred()) - Expect(files).To(HaveLen(1)) - Expect(files[0].IsDir()).To(BeFalse()) + require.NoError(t, err, `error reading %s after stopping [%v]`, debugDir, cmd) + require.Equal(t, 1, len(files), `expected 1 file in %s after stopping [%v]`, debugDir, cmd) - dumpFiles := []string{} + // get a listing of the debug archive + debugArchive := filepath.Join(debugDir, files[0].Name()) + var dumpFiles []string err = archiver.Walk(filepath.Join(debugDir, files[0].Name()), func(f archiver.File) error { dumpFiles = append(dumpFiles, f.Name()) return nil }) - Expect(err).NotTo(HaveOccurred()) - Expect(dumpFiles).To(ContainElements("config_dump.json", "stats.json")) - }, - testCases()..., - ) -}) + require.NoError(t, err, `error reading debug archive %s after stopping [%v]`, debugArchive, cmd) + + // ensure the minimum contents exist + for _, file := range []string{"config_dump.json", "stats.json"} { + require.Contains(t, dumpFiles, file, `debug archive %s doesn't contain %s after stopping [%v]`, debugArchive, file, cmd) + } + }) + } +} diff --git a/test/e2e/getenvoy_extension_test_test.go b/test/e2e/getenvoy_extension_test_test.go index 9bdc2ea5..246833f7 100644 --- a/test/e2e/getenvoy_extension_test_test.go +++ b/test/e2e/getenvoy_extension_test_test.go @@ -15,60 +15,54 @@ package e2e_test import ( - "os" - "path/filepath" + "fmt" + "testing" - . "github.com/onsi/ginkgo" - . "github.com/onsi/ginkgo/extensions/table" - . "github.com/onsi/gomega" + "github.com/stretchr/testify/require" + "github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension" e2e "github.com/tetratelabs/getenvoy/test/e2e/util" ) -var _ = Describe("getenvoy extension test", func() { - - type testCase e2e.CategoryLanguageTuple +// TestGetEnvoyExtensionTest runs the equivalent of "getenvoy extension test" for a matrix of extension.Categories and +// extension.Languages. "getenvoy extension init" is a prerequisite, so run first. +// +// Note: "getenvoy extension test" can be extremely slow due to implicit responsibilities such as downloading modules +// or compilation. This uses Docker, so changes to the Dockerfile or contents like "commands.sh" effect performance. +func TestGetEnvoyExtensionTest(t *testing.T) { + const extensionName = "getenvoy_extension_test" + requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy - testCases := func() []TableEntry { - testCases := make([]TableEntry, 0) - for _, combination := range e2e.GetCategoryLanguageCombinations() { - testCases = append(testCases, Entry(combination.String(), testCase(combination))) - } - return testCases - } + for _, test := range e2e.GetCategoryLanguageCombinations() { + test := test // pin! see https://github.com/kyoh86/scopelint for why - const extensionName = "my.extension" + t.Run(test.String(), func(t *testing.T) { + workDir, removeWorkDir := requireNewTempDir(t) + defer removeWorkDir() - DescribeTable("should run unit tests", - func(given testCase) { - By("choosing the output directory") - outputDir := filepath.Join(tempDir, "new") - defer CleanUpExtensionDir(outputDir) + revertChDir := requireChDir(t, workDir) + defer revertChDir() - By("running `extension init` command") - _, _, err := GetEnvoy("extension init"). - Arg(outputDir). - Arg("--category").Arg(given.Category.String()). - Arg("--language").Arg(given.Language.String()). - Arg("--name").Arg(extensionName). - Exec() - Expect(err).NotTo(HaveOccurred()) + // test requires "get envoy extension init" to have succeeded + requireExtensionInit(t, workDir, test.Category, test.Language, extensionName) + defer requireExtensionClean(t, workDir) - By("changing to the output directory") - err = os.Chdir(outputDir) - Expect(err).NotTo(HaveOccurred()) + cmd := GetEnvoy("extension test").Args(e2e.Env.GetBuiltinContainerOptions()...) + // "getenvoy extension test" only returns stdout because `docker run -t` redirects stderr to stdout. + stdout := requireExecNoStderr(t, cmd) - By("running `extension test` command") - stdout, stderr, err := GetEnvoy("extension test"). - Args(e2e.Env.GetBuiltinContainerOptions()...). - Exec() - Expect(err).NotTo(HaveOccurred()) + // Verify the tests ran + switch test.Language { + case extension.LanguageRust: + // `cargo` colorizes output. After stripping ANSI codes, ensure the output is successful. + stdout = stripAnsiEscapeRegexp.ReplaceAllString(stdout, "") + require.Regexp(t, `(?s)^.*test result: ok.*$`, stdout, `invalid stdout running [%v]`, cmd) - By("verifying stdout/stderr") - // apparently, use of `-t` option in `docker run` causes stderr to be incorporated into stdout - Expect(stdout).NotTo(BeEmpty()) - Expect(stderr).To(BeEmpty()) - }, - testCases()..., - ) -}) + case extension.LanguageTinyGo: + // We expect the test output to include the extension name. + stdoutRegexp := fmt.Sprintf(`(?s)^.*ok %s.*$`, extensionName) + require.Regexp(t, stdoutRegexp, stdout, `invalid stdout running [%v]`, cmd) + } + }) + } +} diff --git a/test/e2e/getenvoy_test.go b/test/e2e/getenvoy_test.go index e3fe246d..4e5bbbf3 100644 --- a/test/e2e/getenvoy_test.go +++ b/test/e2e/getenvoy_test.go @@ -15,18 +15,21 @@ package e2e_test import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "testing" + + "github.com/stretchr/testify/require" ) -var _ = Describe("getenvoy", func() { - It("should support '--version' flag", func() { - By("running the command") - stdout, stderr, err := GetEnvoy("--version").Exec() - Expect(err).NotTo(HaveOccurred()) +func TestGetEnvoyVersion(t *testing.T) { + stdout, stderr, err := GetEnvoy("--version").Exec() + + require.Regexp(t, `^getenvoy version ([^\s]+)\n$`, stdout) + require.Equal(t, ``, stderr) + require.NoError(t, err) +} + +func TestGetEnvoyString(t *testing.T) { + g := GetEnvoy("--version") - By("verifying stdout/stderr") - Expect(stdout).To(MatchRegexp(`^getenvoy version ([^\s]+)\n$`)) - Expect(stderr).To(Equal(``)) - }) -}) + require.Regexp(t, `.*getenvoy --version$`, g.String()) +} diff --git a/test/e2e/util/command.go b/test/e2e/util/command.go index d9c7001a..d55a8973 100644 --- a/test/e2e/util/command.go +++ b/test/e2e/util/command.go @@ -16,10 +16,16 @@ package util import ( "bytes" + "fmt" "io" "os" "os/exec" + "strings" "syscall" + "testing" + "time" + + "github.com/stretchr/testify/require" argutil "github.com/tetratelabs/getenvoy/pkg/util/args" ) @@ -52,6 +58,10 @@ func (b *cmdBuilder) Args(args ...string) *cmdBuilder { return b } +func (b *cmdBuilder) String() string { + return strings.Join(b.cmd.Args, " ") +} + func (b *cmdBuilder) Exec() (string, string, error) { stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) @@ -61,30 +71,37 @@ func (b *cmdBuilder) Exec() (string, string, error) { return stdout.String(), stderr.String(), err } -func (b *cmdBuilder) Start() (io.Reader, io.Reader, func() error, <-chan error) { +func (b *cmdBuilder) Start(t *testing.T, terminateTimeout time.Duration) (io.Reader, io.Reader, func()) { stdout := newSyncBuffer() stderr := newSyncBuffer() b.cmd.Stdout = io.MultiWriter(os.Stdout, stdout) // we want to see full `getenvoy` output in the test log b.cmd.Stderr = io.MultiWriter(os.Stderr, stderr) - errs := make(chan error, 1) - if err := b.cmd.Start(); err != nil { - errs <- err - close(errs) - return stdout, stderr, func() error { return nil }, errs - } + err := b.cmd.Start() + require.NoError(t, err, `error starting [%v]`, b) + + errc := make(chan error, 1) go func() { - defer close(errs) - if err := b.cmd.Wait(); err != nil { - errs <- err - } + errc <- b.cmd.Wait() }() - canceled := false - cancel := func() error { - if canceled { - return nil + + // Ensure terminate is only called once. As this happens from a single thread, there's no need to lock. + terminated := false + terminate := func() { + if terminated { + return + } + terminated = true + + err := b.cmd.Process.Signal(syscall.SIGTERM) + require.NoError(t, err, `error terminating [%v]`, b.cmd) + + select { + case e := <-errc: + require.NoError(t, e, `error running [%v]`, b.cmd) + case <-time.After(terminateTimeout): + t.Fatal(fmt.Sprintf("getenvoy command didn't exit gracefully within %s", terminateTimeout)) } - canceled = true - return b.cmd.Process.Signal(syscall.SIGTERM) } - return stdout, stderr, cancel, errs + + return stdout, stderr, terminate } diff --git a/test/e2e/util/iostream.go b/test/e2e/util/iostream.go index f0bdbdd6..8d1886a8 100644 --- a/test/e2e/util/iostream.go +++ b/test/e2e/util/iostream.go @@ -78,7 +78,7 @@ func StreamLines(r io.Reader) Stream { } } }() - return Stream(lines) + return lines } // NamedStream represents a stream of text lines, such as "stderr" or "stdout". @@ -112,7 +112,7 @@ func (s *NamedStream) FirstMatch(pattern *regexp.Regexp) Single { } match <- StreamError{errors.Errorf("%q didn't have a line that would match %q", s.Name, pattern)} }() - return Single(match) + return match } // Wait waits until a stream emit a line of text or fails if timeout has been exceeded. From 28f5ea66ea737917a94b32fe54891c8d4ead9ae7 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Tue, 30 Mar 2021 19:52:06 +0800 Subject: [PATCH 2/8] typo Signed-off-by: Adrian Cole --- ci/e2e/linux/run_tests.sh | 2 +- ci/e2e/macos/run_tests.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/e2e/linux/run_tests.sh b/ci/e2e/linux/run_tests.sh index 4a5fc7b9..9ef1003a 100755 --- a/ci/e2e/linux/run_tests.sh +++ b/ci/e2e/linux/run_tests.sh @@ -39,4 +39,4 @@ docker-compose up -d export E2E_GETENVOY_BINARY="${WORKSPACE_DIR}/build/bin/linux/amd64/getenvoy" # run e2e tests with '-v' flag to be able to see the progress -go test github.com/tetratelabs/getenvoy/test/e2e -test.parallel 1 -test.failfast -v ./test/e2e/... +go test -test.parallel 1 -test.failfast -v ./test/e2e/... diff --git a/ci/e2e/macos/run_tests.sh b/ci/e2e/macos/run_tests.sh index a5166c13..f8b9998a 100755 --- a/ci/e2e/macos/run_tests.sh +++ b/ci/e2e/macos/run_tests.sh @@ -41,4 +41,4 @@ docker-compose up -d export E2E_GETENVOY_BINARY="${WORKSPACE_DIR}/build/bin/darwin/amd64/getenvoy" # run e2e tests with '-ginkgo.v' flag to be able to see the progress -go test github.com/tetratelabs/getenvoy/test/e2e -test.parallel 1 -test.failfast -v ./test/e2e/... +go test -test.parallel 1 -test.failfast -v ./test/e2e/... From 559a56bc009a78720d396c976eac2f1602d835d5 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Wed, 31 Mar 2021 08:20:18 +0800 Subject: [PATCH 3/8] Increase default timeout, reuse envoy home in 'go extension run' commands Signed-off-by: Adrian Cole --- Makefile | 3 +- ci/e2e/macos/run_tests.sh | 2 +- test/e2e/getenvoy_extension_build_test.go | 3 ++ test/e2e/getenvoy_extension_push_test.go | 3 ++ test/e2e/getenvoy_extension_run_test.go | 53 +++++++++++++++++++---- test/e2e/getenvoy_extension_test_test.go | 3 ++ 6 files changed, 56 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 22c9ecf4..f68faa5b 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,8 @@ GO_COVERAGE_OPTS ?= -covermode=atomic -coverpkg=./... GO_COVERAGE_EXTRA_OPTS ?= E2E_PKG_LIST ?= ./test/e2e/... -E2E_OPTS ?= -test.parallel 1 -test.failfast +# Set the default timeout >10m as particularly rust e2e tests are slow https://golang.org/cmd/go/#hdr-Testing_flags +E2E_OPTS ?= -timeout 45m -test.parallel 1 -test.failfast E2E_EXTRA_OPTS ?= GOOSES := linux darwin diff --git a/ci/e2e/macos/run_tests.sh b/ci/e2e/macos/run_tests.sh index f8b9998a..f5e2cf4e 100755 --- a/ci/e2e/macos/run_tests.sh +++ b/ci/e2e/macos/run_tests.sh @@ -28,7 +28,7 @@ mkdir -p "${E2E_CACHE_DIR}" # to speed up `getenvoy extension build|test`, re-use a single cache across all extensions created by e2e tests export E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS} -v ${E2E_CACHE_DIR}:/tmp/cache/getenvoy -e CARGO_HOME=/tmp/cache/getenvoy/extension/rust-builder/cargo" -# set HOME directory +# set HOME directory (TODO: why? and also why not GETENVOY_HOME?) export E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS} -e HOME=/tmp/getenvoy" # restore executable bit that get lost by Github Actions during artifact upload/download diff --git a/test/e2e/getenvoy_extension_build_test.go b/test/e2e/getenvoy_extension_build_test.go index 9a5a1f45..c950a7d0 100644 --- a/test/e2e/getenvoy_extension_build_test.go +++ b/test/e2e/getenvoy_extension_build_test.go @@ -28,6 +28,9 @@ import ( // // Note: "getenvoy extension build" can be extremely slow due to implicit responsibilities such as downloading modules // or compilation. This uses Docker, so changes to the Dockerfile or contents like "commands.sh" effect performance. +// +// Note: Pay close attention to values of util.E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS as these can change assumptions. +// CI may override this to set HOME or CARGO_HOME (rust) used by "getenvoy" and effect its execution. func TestGetEnvoyExtensionBuild(t *testing.T) { const extensionName = "getenvoy_extension_build" requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy diff --git a/test/e2e/getenvoy_extension_push_test.go b/test/e2e/getenvoy_extension_push_test.go index d553525f..b96d3e1d 100644 --- a/test/e2e/getenvoy_extension_push_test.go +++ b/test/e2e/getenvoy_extension_push_test.go @@ -34,6 +34,9 @@ import ( // // Note: "getenvoy extension build" can be extremely slow due to implicit responsibilities such as downloading modules // or compilation. This uses Docker, so changes to the Dockerfile or contents like "commands.sh" effect performance. +// +// Note: Pay close attention to values of util.E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS as these can change assumptions. +// CI may override this to set HOME or CARGO_HOME (rust) used by "getenvoy" and effect its execution. func TestGetEnvoyExtensionPush(t *testing.T) { const extensionName = "getenvoy_extension_push" // localRegistryWasmImageRef corresponds to a Docker container running the image "registry:2" diff --git a/test/e2e/getenvoy_extension_run_test.go b/test/e2e/getenvoy_extension_run_test.go index b5a4b98d..ef2584ea 100644 --- a/test/e2e/getenvoy_extension_run_test.go +++ b/test/e2e/getenvoy_extension_run_test.go @@ -16,6 +16,7 @@ package e2e_test import ( "io/ioutil" + "os" "path/filepath" "regexp" "testing" @@ -25,29 +26,32 @@ import ( "github.com/stretchr/testify/require" "github.com/tetratelabs/log" + "github.com/tetratelabs/getenvoy/pkg/common" e2e "github.com/tetratelabs/getenvoy/test/e2e/util" utilenvoy "github.com/tetratelabs/getenvoy/test/e2e/util/envoy" ) +const extensionName = "getenvoy_extension_run" +const terminateTimeout = 2 * time.Minute + // TestGetEnvoyExtensionRun runs the equivalent of "getenvoy extension run" for a matrix of extension.Categories and // extension.Languages. "getenvoy extension init" is a prerequisite, so run first. // // Note: "getenvoy extension run" can be extremely slow due to implicit responsibilities such as downloading modules // or compilation. This uses Docker, so changes to the Dockerfile or contents like "commands.sh" effect performance. +// +// Note: Pay close attention to values of util.E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS as these can change assumptions. +// CI may override this to set HOME or CARGO_HOME (rust) used by "getenvoy" and effect directories used here. func TestGetEnvoyExtensionRun(t *testing.T) { - const extensionName = "getenvoy_extension_run" - const terminateTimeout = 2 * time.Minute requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy + debugDir, revertOriginalDebugDir := backupDebugDir(t) + defer revertOriginalDebugDir() + for _, test := range e2e.GetCategoryLanguageCombinations() { test := test // pin! see https://github.com/kyoh86/scopelint for why t.Run(test.String(), func(t *testing.T) { - // We override the home directory ~/.getenvoy so as to not overwrite any debug info there. We need to do - // this to ensure presence of debug files are from this run, not another one. - homeDir, removeHomeDir := requireNewTempDir(t) - defer removeHomeDir() - workDir, removeWorkDir := requireNewTempDir(t) defer removeWorkDir() @@ -60,7 +64,6 @@ func TestGetEnvoyExtensionRun(t *testing.T) { // "getenvoy extension run" only returns stdout because `docker run -t` redirects stderr to stdout. cmd := GetEnvoy("extension run --envoy-options '-l trace'"). - Args("--home-dir", homeDir). Args(e2e.Env.GetBuiltinContainerOptions()...) _, stderr, terminate := cmd.Start(t, terminateTimeout) defer terminate() @@ -101,10 +104,13 @@ func TestGetEnvoyExtensionRun(t *testing.T) { terminate() // verify the debug dump of Envoy state has been taken - debugDir := filepath.Join(homeDir, "debug") files, err := ioutil.ReadDir(debugDir) require.NoError(t, err, `error reading %s after stopping [%v]`, debugDir, cmd) require.Equal(t, 1, len(files), `expected 1 file in %s after stopping [%v]`, debugDir, cmd) + defer func() { + e := os.RemoveAll(debugDir) + require.NoError(t, e, `error removing debug dir %s after stopping [%v]`, debugDir, cmd) + }() // get a listing of the debug archive debugArchive := filepath.Join(debugDir, files[0].Name()) @@ -122,3 +128,32 @@ func TestGetEnvoyExtensionRun(t *testing.T) { }) } } + +// backupDebugDir backs up ${GETENVOY_HOME}/debug in case the test hasn't overridden it and the developer has existing +// data there. The function returned reverts this directory. +// +// Typically, this will run in the default ~/.getenvoy directory, as a means to avoid re-downloads of files such as +// .getenvoy/builds/standard/1.17.0/darwin/bin/envoy (~100MB) +// +// While CI usually overrides the `HOME` variable with E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS, a developer may be +// running this on their laptop. To avoid clobbering their old debug data, backup the +func backupDebugDir(t *testing.T) (string, func()) { + debugDir := filepath.Join(common.HomeDir, "debug") + + if _, err := os.Lstat(debugDir); err != nil && os.IsNotExist(err) { + return debugDir, func() {} // do nothing on remove, if there was no debug backup + } + + // get a name of a new temp directory, which we'll rename the existing debug to + backupDir, _ := requireNewTempDir(t) + err := os.RemoveAll(backupDir) + require.NoError(t, err, `error removing temp directory: %s`, backupDir) + + err = os.Rename(debugDir, backupDir) + require.NoError(t, err, `error renaming debug dir %s to %s`, debugDir, backupDir) + + return debugDir, func() { + err = os.Rename(backupDir, debugDir) + require.NoError(t, err, `error renaming backup dir %s to %s`, debugDir, backupDir) + } +} diff --git a/test/e2e/getenvoy_extension_test_test.go b/test/e2e/getenvoy_extension_test_test.go index 246833f7..42616e03 100644 --- a/test/e2e/getenvoy_extension_test_test.go +++ b/test/e2e/getenvoy_extension_test_test.go @@ -29,6 +29,9 @@ import ( // // Note: "getenvoy extension test" can be extremely slow due to implicit responsibilities such as downloading modules // or compilation. This uses Docker, so changes to the Dockerfile or contents like "commands.sh" effect performance. +// +// Note: Pay close attention to values of util.E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS as these can change assumptions. +// CI may override this to set GETENVOY_HOME or CARGO_HOME (rust) used by "getenvoy" and effect its execution. func TestGetEnvoyExtensionTest(t *testing.T) { const extensionName = "getenvoy_extension_test" requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy From 613a7dadd4e4648e6800590f946bd91dfaa70d11 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Wed, 31 Mar 2021 08:23:52 +0800 Subject: [PATCH 4/8] make e2e should be similar to what CI does Signed-off-by: Adrian Cole --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f68faa5b..ed766d97 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,8 @@ GO_COVERAGE_EXTRA_OPTS ?= E2E_PKG_LIST ?= ./test/e2e/... # Set the default timeout >10m as particularly rust e2e tests are slow https://golang.org/cmd/go/#hdr-Testing_flags -E2E_OPTS ?= -timeout 45m -test.parallel 1 -test.failfast +# Run only one test at a time, in verbose mode, so that failures are easy to diagnose. Stop at first error. +E2E_OPTS ?= -timeout 45m -test.parallel 1 -v -test.failfast E2E_EXTRA_OPTS ?= GOOSES := linux darwin From 8e77fc7bb77059e9fdbd337ee9d2f5883896ee66 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Wed, 31 Mar 2021 09:14:13 +0800 Subject: [PATCH 5/8] Makes sure important parameters aren't duplicated in make and shell scripts Signed-off-by: Adrian Cole --- .github/workflows/commit.yaml | 8 ++++++-- .github/workflows/release.yaml | 4 ++-- ci/e2e/linux/run_tests.sh | 18 ++++-------------- ci/e2e/macos/run_tests.sh | 20 +++++--------------- 4 files changed, 17 insertions(+), 33 deletions(-) diff --git a/.github/workflows/commit.yaml b/.github/workflows/commit.yaml index 8dacf911..870d4fc4 100644 --- a/.github/workflows/commit.yaml +++ b/.github/workflows/commit.yaml @@ -62,7 +62,9 @@ jobs: run: make builders - name: "Run e2e tests using the `getenvoy` binary built by the upstream job" - run: ./ci/e2e/linux/run_tests.sh + run: | # chmod to restore permissions lost in actions/download-artifact@v2 + chmod a+x build/bin/linux/amd64/getenvoy + ./ci/e2e/macos/run_tests.sh e2e_macos: name: "Run e2e tests on MacOS" @@ -93,4 +95,6 @@ jobs: timeout-minutes: 10 # fail fast if MacOS runner becomes to slow - name: "Run e2e tests using the `getenvoy` binary built by the upstream job" - run: ./ci/e2e/macos/run_tests.sh + run: | # chmod to restore permissions lost in actions/download-artifact@v2 + chmod a+x build/bin/linux/amd64/getenvoy + ./ci/e2e/macos/run_tests.sh diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 54527e8b..592029b8 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -99,7 +99,7 @@ jobs: env: INPUT_FILE: getenvoy_${{ env.RELEASE_VERSION }}_Linux_x86_64.tar.gz INPUT_VERSION: tags/${{ env.RELEASE_TAG }} - run: | + run: | # extract getenvoy to location used in `make e2e`. don't chmod because tar.gz should be correct. curl -s https://raw.githubusercontent.com/dsaltares/fetch-gh-release-asset/0.0.5/fetch_github_asset.sh | bash mkdir -p build/bin/linux/amd64 tar -C build/bin/linux/amd64 -xf ${INPUT_FILE} getenvoy @@ -139,7 +139,7 @@ jobs: env: INPUT_FILE: getenvoy_${{ env.RELEASE_VERSION }}_Darwin_x86_64.tar.gz INPUT_VERSION: tags/${{ env.RELEASE_TAG }} - run: | + run: | # extract getenvoy to location used in `make e2e`. don't chmod because tar.gz should be correct. curl -s https://raw.githubusercontent.com/dsaltares/fetch-gh-release-asset/0.0.5/fetch_github_asset.sh | bash mkdir -p build/bin/darwin/amd64 tar -C build/bin/darwin/amd64 -xf ${INPUT_FILE} getenvoy diff --git a/ci/e2e/linux/run_tests.sh b/ci/e2e/linux/run_tests.sh index 9ef1003a..2f56da58 100755 --- a/ci/e2e/linux/run_tests.sh +++ b/ci/e2e/linux/run_tests.sh @@ -16,9 +16,8 @@ set -e -SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}")" && pwd) -WORKSPACE_DIR="${SCRIPT_DIR}/../../.." - +# see #153 as some of the below may be incorporated into make or CI setup or otherwise improved +# TODO: why default to /tmp while macos defaults to $HOME? E2E_CACHE_DIR="${E2E_CACHE_DIR:-/tmp/cache/getenvoy}" # make sure the cache directory is first created on behalf of the current user @@ -29,14 +28,5 @@ sudo chown -R $(id -u):$(id -g) "${E2E_CACHE_DIR}" # to speed up `getenvoy extension build|test`, re-use a single cache across all extensions created by e2e tests export E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS} -v ${E2E_CACHE_DIR}:/tmp/cache/getenvoy -e CARGO_HOME=/tmp/cache/getenvoy/extension/rust-builder/cargo" -# restore executable bit that get lost by Github Actions during artifact upload/download -chmod a+x ${WORKSPACE_DIR}/build/bin/linux/amd64/* - -# start other containers required in e2e tests -docker-compose up -d - -# run e2e tests on a `getenvoy` binary built by the upstream job -export E2E_GETENVOY_BINARY="${WORKSPACE_DIR}/build/bin/linux/amd64/getenvoy" - -# run e2e tests with '-v' flag to be able to see the progress -go test -test.parallel 1 -test.failfast -v ./test/e2e/... +# run the normal make script. +make e2e diff --git a/ci/e2e/macos/run_tests.sh b/ci/e2e/macos/run_tests.sh index f5e2cf4e..b55d96aa 100755 --- a/ci/e2e/macos/run_tests.sh +++ b/ci/e2e/macos/run_tests.sh @@ -16,9 +16,8 @@ set -e -SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}")" && pwd) -WORKSPACE_DIR="${SCRIPT_DIR}/../../.." - +# see #153 as some of the below may be incorporated into make or CI setup or otherwise improved +# TODO: why default to HOME where linux defaults to /tmp? E2E_CACHE_DIR="${E2E_CACHE_DIR:-$HOME/cache/getenvoy}" # make sure the cache directory is first created on behalf of the current user @@ -28,17 +27,8 @@ mkdir -p "${E2E_CACHE_DIR}" # to speed up `getenvoy extension build|test`, re-use a single cache across all extensions created by e2e tests export E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS} -v ${E2E_CACHE_DIR}:/tmp/cache/getenvoy -e CARGO_HOME=/tmp/cache/getenvoy/extension/rust-builder/cargo" -# set HOME directory (TODO: why? and also why not GETENVOY_HOME?) +# set HOME directory (TODO: why? why not GETENVOY_HOME? why not also in linux?) export E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS} -e HOME=/tmp/getenvoy" -# restore executable bit that get lost by Github Actions during artifact upload/download -chmod a+x ${WORKSPACE_DIR}/build/bin/darwin/amd64/* - -# start other containers required in e2e tests -docker-compose up -d - -# run e2e tests on a `getenvoy` binary built by the upstream job -export E2E_GETENVOY_BINARY="${WORKSPACE_DIR}/build/bin/darwin/amd64/getenvoy" - -# run e2e tests with '-ginkgo.v' flag to be able to see the progress -go test -test.parallel 1 -test.failfast -v ./test/e2e/... +# run the normal make script. +make e2e From 03bb56d1a56ddb33e9d3d44b36ec07f7e2c8942f Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Wed, 31 Mar 2021 11:33:26 +0800 Subject: [PATCH 6/8] Uses TestMain to consolidate variable enforcement Signed-off-by: Adrian Cole --- test/e2e/getenvoy_extension_build_test.go | 7 +---- test/e2e/getenvoy_extension_examples_test.go | 3 +- test/e2e/getenvoy_extension_init_test.go | 3 +- test/e2e/getenvoy_extension_push_test.go | 8 ++--- test/e2e/getenvoy_extension_run_test.go | 8 +---- test/e2e/getenvoy_extension_test_test.go | 7 +---- test/e2e/{e2e_test.go => main_test.go} | 31 +++++++++++++++----- 7 files changed, 30 insertions(+), 37 deletions(-) rename test/e2e/{e2e_test.go => main_test.go} (82%) diff --git a/test/e2e/getenvoy_extension_build_test.go b/test/e2e/getenvoy_extension_build_test.go index c950a7d0..9a901315 100644 --- a/test/e2e/getenvoy_extension_build_test.go +++ b/test/e2e/getenvoy_extension_build_test.go @@ -26,14 +26,9 @@ import ( // TestGetEnvoyExtensionBuild runs the equivalent of "getenvoy extension build" for a matrix of extension.Categories and // extension.Languages. "getenvoy extension init" is a prerequisite, so run first. // -// Note: "getenvoy extension build" can be extremely slow due to implicit responsibilities such as downloading modules -// or compilation. This uses Docker, so changes to the Dockerfile or contents like "commands.sh" effect performance. -// -// Note: Pay close attention to values of util.E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS as these can change assumptions. -// CI may override this to set HOME or CARGO_HOME (rust) used by "getenvoy" and effect its execution. +// "getenvoy extension build" uses Docker. See TestMain for general notes on about the test runtime. func TestGetEnvoyExtensionBuild(t *testing.T) { const extensionName = "getenvoy_extension_build" - requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy for _, test := range e2e.GetCategoryLanguageCombinations() { test := test // pin! see https://github.com/kyoh86/scopelint for why diff --git a/test/e2e/getenvoy_extension_examples_test.go b/test/e2e/getenvoy_extension_examples_test.go index 42fa08b1..148aa701 100644 --- a/test/e2e/getenvoy_extension_examples_test.go +++ b/test/e2e/getenvoy_extension_examples_test.go @@ -28,10 +28,9 @@ import ( // TestGetEnvoyExtensionExampleAdd runs the equivalent of "getenvoy extension example XXX" commands for a matrix of // extension.Categories and extension.Languages. // -// "getenvoy extension example XXX" should be fast and reliable because they don't use Docker. +// "getenvoy extension example" does not use Docker. See TestMain for general notes on about the test runtime. func TestGetEnvoyExtensionExample(t *testing.T) { const extensionName = "getenvoy_extension_example" - requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy for _, test := range e2e.GetCategoryLanguageCombinations() { test := test // pin! see https://github.com/kyoh86/scopelint for why diff --git a/test/e2e/getenvoy_extension_init_test.go b/test/e2e/getenvoy_extension_init_test.go index f6bd5d42..28479580 100644 --- a/test/e2e/getenvoy_extension_init_test.go +++ b/test/e2e/getenvoy_extension_init_test.go @@ -30,10 +30,9 @@ import ( // TestGetEnvoyExtensionInit runs the equivalent of "getenvoy extension init" for a matrix of extension.Categories and // extension.Languages. // -// "getenvoy extension init" should be fast and reliable because it doesn't use Docker. +// "getenvoy extension init" does not use Docker. See TestMain for general notes on about the test runtime. func TestGetEnvoyExtensionInit(t *testing.T) { const extensionName = "getenvoy_extension_init" - requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy type testTuple struct { testName string diff --git a/test/e2e/getenvoy_extension_push_test.go b/test/e2e/getenvoy_extension_push_test.go index b96d3e1d..c609afb7 100644 --- a/test/e2e/getenvoy_extension_push_test.go +++ b/test/e2e/getenvoy_extension_push_test.go @@ -32,18 +32,14 @@ import ( // This test does not attempt to use the image built as that would be redundant to other tests. Rather, this focuses on // whether we can read back exactly what was pushed to the registry. // -// Note: "getenvoy extension build" can be extremely slow due to implicit responsibilities such as downloading modules -// or compilation. This uses Docker, so changes to the Dockerfile or contents like "commands.sh" effect performance. -// -// Note: Pay close attention to values of util.E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS as these can change assumptions. -// CI may override this to set HOME or CARGO_HOME (rust) used by "getenvoy" and effect its execution. +// "getenvoy extension run" depends on a Docker container, and "getenvoy extension build" uses Docker. +// See TestMain for general notes on about the test runtime. func TestGetEnvoyExtensionPush(t *testing.T) { const extensionName = "getenvoy_extension_push" // localRegistryWasmImageRef corresponds to a Docker container running the image "registry:2" const localRegistryWasmImageRef = "localhost:5000/getenvoy/" + extensionName // When unspecified, we default the tag to Docker's default "latest". Note: recent tools enforce qualifying this! const defaultTag = "latest" - requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy type testTuple struct { name string diff --git a/test/e2e/getenvoy_extension_run_test.go b/test/e2e/getenvoy_extension_run_test.go index ef2584ea..2aaaa65a 100644 --- a/test/e2e/getenvoy_extension_run_test.go +++ b/test/e2e/getenvoy_extension_run_test.go @@ -37,14 +37,8 @@ const terminateTimeout = 2 * time.Minute // TestGetEnvoyExtensionRun runs the equivalent of "getenvoy extension run" for a matrix of extension.Categories and // extension.Languages. "getenvoy extension init" is a prerequisite, so run first. // -// Note: "getenvoy extension run" can be extremely slow due to implicit responsibilities such as downloading modules -// or compilation. This uses Docker, so changes to the Dockerfile or contents like "commands.sh" effect performance. -// -// Note: Pay close attention to values of util.E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS as these can change assumptions. -// CI may override this to set HOME or CARGO_HOME (rust) used by "getenvoy" and effect directories used here. +// "getenvoy extension run" uses Docker. See TestMain for general notes on about the test runtime. func TestGetEnvoyExtensionRun(t *testing.T) { - requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy - debugDir, revertOriginalDebugDir := backupDebugDir(t) defer revertOriginalDebugDir() diff --git a/test/e2e/getenvoy_extension_test_test.go b/test/e2e/getenvoy_extension_test_test.go index 42616e03..8dac71c5 100644 --- a/test/e2e/getenvoy_extension_test_test.go +++ b/test/e2e/getenvoy_extension_test_test.go @@ -27,14 +27,9 @@ import ( // TestGetEnvoyExtensionTest runs the equivalent of "getenvoy extension test" for a matrix of extension.Categories and // extension.Languages. "getenvoy extension init" is a prerequisite, so run first. // -// Note: "getenvoy extension test" can be extremely slow due to implicit responsibilities such as downloading modules -// or compilation. This uses Docker, so changes to the Dockerfile or contents like "commands.sh" effect performance. -// -// Note: Pay close attention to values of util.E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS as these can change assumptions. -// CI may override this to set GETENVOY_HOME or CARGO_HOME (rust) used by "getenvoy" and effect its execution. +// "getenvoy extension test" uses Docker. See TestMain for general notes on about the test runtime. func TestGetEnvoyExtensionTest(t *testing.T) { const extensionName = "getenvoy_extension_test" - requireEnvoyBinaryPath(t) // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy for _, test := range e2e.GetCategoryLanguageCombinations() { test := test // pin! see https://github.com/kyoh86/scopelint for why diff --git a/test/e2e/e2e_test.go b/test/e2e/main_test.go similarity index 82% rename from test/e2e/e2e_test.go rename to test/e2e/main_test.go index 9ada6f40..8ecd873a 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/main_test.go @@ -15,6 +15,7 @@ package e2e_test import ( + "fmt" "io/ioutil" "os" "path/filepath" @@ -31,16 +32,30 @@ import ( var ( // GetEnvoy is a convenient alias. GetEnvoy = e2e.GetEnvoy -) -// stripAnsiEscapeRegexp is a regular expression to clean ANSI Control sequences -// feat https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python#33925425 -var stripAnsiEscapeRegexp = regexp.MustCompile(`(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]`) + // stripAnsiEscapeRegexp is a regular expression to clean ANSI Control sequences + // feat https://stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python#33925425 + stripAnsiEscapeRegexp = regexp.MustCompile(`(\x9B|\x1B\[)[0-?]*[ -/]*[@-~]`) +) -func requireEnvoyBinaryPath(t *testing.T) { - path, err := e2e.Env.GetEnvoyBinary() - require.NoError(t, err, `error reading path to getenvoy binary`) - e2e.GetEnvoyBinaryPath = path +// TestMain ensures state required for all tests, notably that util.E2E_GETENVOY_BINARY is set. +// +// Note: "getenvoy extension build" and commands that imply it, can be extremely slow due to implicit responsibilities +// such as downloading modules or compilation. Commands like this use Docker, so changes to the Dockerfile or contents +// like "commands.sh" will effect performance. +// +// Note: Pay close attention to values of util.E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS as these can change assumptions. +// CI may override this to set HOME or CARGO_HOME (rust) used by "getenvoy" and effect its execution. +func TestMain(m *testing.M) { + // As this is an e2e test, we execute all tests with a binary compiled earlier. + // + // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy + _, err := e2e.Env.GetEnvoyBinary() + if err != nil { + fmt.Fprintf(os.Stderr, `failed to start e2e tests: %v`, err) + os.Exit(1) + } + os.Exit(m.Run()) } // requireNewTempDir creates a new directory. The function returned cleans it up. From 9f7d7be33dd72955acf2a2cc16a56adc29141a8a Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Wed, 31 Mar 2021 11:59:36 +0800 Subject: [PATCH 7/8] whoops Signed-off-by: Adrian Cole --- test/e2e/main_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 8ecd873a..05830faa 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -50,11 +50,12 @@ func TestMain(m *testing.M) { // As this is an e2e test, we execute all tests with a binary compiled earlier. // // Ex. After running "make bin", E2E_GETENVOY_BINARY=$PWD/build/bin/darwin/amd64/getenvoy - _, err := e2e.Env.GetEnvoyBinary() + path, err := e2e.Env.GetEnvoyBinary() if err != nil { fmt.Fprintf(os.Stderr, `failed to start e2e tests: %v`, err) os.Exit(1) } + e2e.GetEnvoyBinaryPath = path os.Exit(m.Run()) } From 8c5b32ad0b8b419ab7ffb119d0ebe4005e0a0478 Mon Sep 17 00:00:00 2001 From: Adrian Cole Date: Wed, 31 Mar 2021 12:43:05 +0800 Subject: [PATCH 8/8] moves up terminate pre-emption Signed-off-by: Adrian Cole --- test/e2e/getenvoy_extension_run_test.go | 10 +++++++++- test/e2e/util/command.go | 11 +---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/test/e2e/getenvoy_extension_run_test.go b/test/e2e/getenvoy_extension_run_test.go index 2aaaa65a..8334168b 100644 --- a/test/e2e/getenvoy_extension_run_test.go +++ b/test/e2e/getenvoy_extension_run_test.go @@ -60,7 +60,12 @@ func TestGetEnvoyExtensionRun(t *testing.T) { cmd := GetEnvoy("extension run --envoy-options '-l trace'"). Args(e2e.Env.GetBuiltinContainerOptions()...) _, stderr, terminate := cmd.Start(t, terminateTimeout) - defer terminate() + + // The underlying call is conditional to ensure errors that raise before we stop the server, stop it. + deferredTerminate := terminate + defer func() { + deferredTerminate() + }() stderrLines := e2e.StreamLines(stderr).Named("stderr") @@ -96,6 +101,9 @@ func TestGetEnvoyExtensionRun(t *testing.T) { log.Infof(`stopping Envoy after running [%v]`, cmd) terminate() + deferredTerminate = func() { + // no-op as we already terminated + } // verify the debug dump of Envoy state has been taken files, err := ioutil.ReadDir(debugDir) diff --git a/test/e2e/util/command.go b/test/e2e/util/command.go index d55a8973..dc767507 100644 --- a/test/e2e/util/command.go +++ b/test/e2e/util/command.go @@ -84,14 +84,7 @@ func (b *cmdBuilder) Start(t *testing.T, terminateTimeout time.Duration) (io.Rea errc <- b.cmd.Wait() }() - // Ensure terminate is only called once. As this happens from a single thread, there's no need to lock. - terminated := false - terminate := func() { - if terminated { - return - } - terminated = true - + return stdout, stderr, func() { err := b.cmd.Process.Signal(syscall.SIGTERM) require.NoError(t, err, `error terminating [%v]`, b.cmd) @@ -102,6 +95,4 @@ func (b *cmdBuilder) Start(t *testing.T, terminateTimeout time.Duration) (io.Rea t.Fatal(fmt.Sprintf("getenvoy command didn't exit gracefully within %s", terminateTimeout)) } } - - return stdout, stderr, terminate }