Pulls E2E_EXTENSION_LANGUAGE into an external matrix

Before, end-to-end test executions were very slow. Originally, there was
only one extension language. When this changed, the e2e tests took
longer than before, and there was no quick way to run all tests when
only changing one language. This is because the test case managed the
matrix of extension language.

The change here is to pull the extension language out to a different ENV
variable: `E2E_EXTENSION_LANGUAGE`. CI now manages this externally,
which allows quick identification of errors or performance issues on a
per-language basis. It also increases the incentive to run `make e2e`,
by removing the "language I'm not using" penalty.

It is understood that currently end-to-end tests happen to only test
extensions and that this optimizes for the current codebase, not a
future one that may have other matrix concerns. If such a topic happens
later, we can consider alternate approaches, and meanwhile enjoy the
isolation.

See #152

Signed-off-by: Adrian Cole <adrian@tetrate.io>

.github/workflows/commit.yaml

@@ -48,6 +48,13 @@ jobs:
       - bin
     runs-on: ubuntu-latest
     timeout-minutes: 30  # instead of 360 by default
+    strategy:
+      matrix:
+        include:  # Controls language used in all extension tests. Managed by test/e2e/main_test.go
+          - extension-language: rust
+          - extension-language: tinygo
+    env:
+      E2E_EXTENSION_LANGUAGE: ${{ matrix.extension-language }}
     steps:
       - name: "Checkout"
         uses: actions/checkout@v2
@@ -72,6 +79,13 @@ jobs:
       - bin
     runs-on: macos-latest
     timeout-minutes: 90  # instead of 360 by default
+    strategy:
+      matrix:
+        include:  # Controls language used in all extension tests. Managed by test/e2e/main_test.go
+          - extension-language: rust
+          - extension-language: tinygo
+    env:
+      E2E_EXTENSION_LANGUAGE: ${{ matrix.extension-language }}
     steps:
       - name: "Checkout"
         uses: actions/checkout@v2

.github/workflows/release.yaml

@@ -78,6 +78,13 @@ jobs:
       - builders
     runs-on: ubuntu-latest
     timeout-minutes: 30  # instead of 360 by default
+    strategy:
+      matrix:
+        include:  # Controls language used in all extension tests. Managed by test/e2e/main_test.go
+          - extension-language: rust
+          - extension-language: tinygo
+    env:
+      E2E_EXTENSION_LANGUAGE: ${{ matrix.extension-language }}
     steps:
       - name: "Checkout"
         uses: actions/checkout@v2
@@ -95,7 +102,7 @@ jobs:
           name: bin
           path: build/bin
 
-      - name: "Download `getenvoy` binary from GithHub release assets"
+      - name: "Download `getenvoy` binary from GitHub release assets"
         env:
           INPUT_FILE: getenvoy_${{ env.RELEASE_VERSION }}_Linux_x86_64.tar.gz
           INPUT_VERSION: tags/${{ env.RELEASE_TAG }}
@@ -118,6 +125,13 @@ jobs:
       - builders
     runs-on: macos-latest
     timeout-minutes: 90  # instead of 360 by default
+    strategy:
+      matrix:
+        include:  # Controls language used in all extension tests. Managed by test/e2e/main_test.go
+          - extension-language: rust
+          - extension-language: tinygo
+    env:
+      E2E_EXTENSION_LANGUAGE: ${{ matrix.extension-language }}
     steps:
       - name: "Checkout"
         uses: actions/checkout@v2
@@ -135,7 +149,7 @@ jobs:
           name: bin
           path: build/bin
 
-      - name: "Download `getenvoy` binary from GithHub release assets"
+      - name: "Download `getenvoy` binary from GitHub release assets"
         env:
           INPUT_FILE: getenvoy_${{ env.RELEASE_VERSION }}_Darwin_x86_64.tar.gz
           INPUT_VERSION: tags/${{ env.RELEASE_TAG }}

DEVELOPER.md

@@ -44,7 +44,13 @@ and follow the wizard.
 
 ### How to run e2e Tests
 
-Run:
-```shell
-make e2e
-```
+End-to-end (e2e) tests rely on a `getenvoy` binary that defaults to what was built by `make bin`.
+
+In simplest case, execute `make e2e` to run all tests configured. You can also set the below ENV variables to effect the
+runtime. These are defined in [main_test.go](test/e2e/main_test.go).
+
+Environment Variable              | Description
+--------------------------------- | ------------------------------------------------------------------------------------
+`E2E_GETENVOY_BINARY`             | Overrides `getenvoy` binary. Defaults to `$PWD/build/bin/$GOOS/$GOARCH/getenvoy`
+`E2E_TOOLCHAIN_CONTAINER_OPTIONS` | Overrides `--toolchain-container-options` in Docker commands. Defaults to "".
+`E2E_EXTENSION_LANGUAGE`          | Overrides `--language` in `getenvoy extension` commands. Defaults to "tinygo".

ci/e2e/linux/run_tests.sh

@@ -26,7 +26,7 @@ mkdir -p "${E2E_CACHE_DIR}"
 sudo chown -R $(id -u):$(id -g) "${E2E_CACHE_DIR}"
 
 # to speed up `getenvoy extension build|test`, re-use a single cache across all extensions created by e2e tests
-export E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS} -v ${E2E_CACHE_DIR}:/tmp/cache/getenvoy -e CARGO_HOME=/tmp/cache/getenvoy/extension/rust-builder/cargo"
+export E2E_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_TOOLCHAIN_CONTAINER_OPTIONS} -v ${E2E_CACHE_DIR}:/tmp/cache/getenvoy -e CARGO_HOME=/tmp/cache/getenvoy/extension/rust-builder/cargo"
 
 # run the normal make script.
 make e2e

ci/e2e/macos/run_tests.sh

@@ -25,10 +25,10 @@ mkdir -p "${E2E_CACHE_DIR}"
 
 # TODO: support multiple language
 # to speed up `getenvoy extension build|test`, re-use a single cache across all extensions created by e2e tests
-export E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS} -v ${E2E_CACHE_DIR}:/tmp/cache/getenvoy -e CARGO_HOME=/tmp/cache/getenvoy/extension/rust-builder/cargo"
+export E2E_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_TOOLCHAIN_CONTAINER_OPTIONS} -v ${E2E_CACHE_DIR}:/tmp/cache/getenvoy -e CARGO_HOME=/tmp/cache/getenvoy/extension/rust-builder/cargo"
 
 # set HOME directory (TODO: why? why not GETENVOY_HOME? why not also in linux?)
-export E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS} -e HOME=/tmp/getenvoy"
+export E2E_TOOLCHAIN_CONTAINER_OPTIONS="${E2E_TOOLCHAIN_CONTAINER_OPTIONS} -e HOME=/tmp/getenvoy"
 
 # run the normal make script.
 make e2e

test/e2e/getenvoy_extension_build_test.go

@@ -20,37 +20,37 @@ import (
 
 	"github.com/stretchr/testify/require"
 
-	e2e "github.com/tetratelabs/getenvoy/test/e2e/util"
+	"github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension"
 )
 
-// TestGetEnvoyExtensionBuild runs the equivalent of "getenvoy extension build" for a matrix of extension.Categories and
-// extension.Languages. "getenvoy extension init" is a prerequisite, so run first.
+// TestGetEnvoyExtensionBuild runs the equivalent of "getenvoy extension build" for a matrix of extension.Categories.
+// "getenvoy extension init" is a prerequisite, so run first.
 //
 // "getenvoy extension build" uses Docker. See TestMain for general notes on about the test runtime.
 func TestGetEnvoyExtensionBuild(t *testing.T) {
 	const extensionName = "getenvoy_extension_build"
 
-	for _, test := range e2e.GetCategoryLanguageCombinations() {
-		test := test // pin! see https://github.com/kyoh86/scopelint for why
+	for _, category := range extension.Categories {
+		category := category // pin! see https://github.com/kyoh86/scopelint for why
 
-		t.Run(test.String(), func(t *testing.T) {
+		t.Run(category.String(), func(t *testing.T) {
 			workDir, removeWorkDir := requireNewTempDir(t)
 			defer removeWorkDir()
 
 			revertChDir := requireChDir(t, workDir)
 			defer revertChDir()
 
 			// test requires "get envoy extension init" to have succeeded
-			requireExtensionInit(t, workDir, test.Category, test.Language, extensionName)
+			requireExtensionInit(t, workDir, category, extensionName)
 			defer requireExtensionClean(t, workDir)
 
 			// "getenvoy extension build" only returns stdout because `docker run -t` redirects stderr to stdout.
 			// We don't verify stdout because it is low signal vs looking at files created.
-			cmd := GetEnvoy("extension build").Args(getBuiltinContainerOptions()...)
+			cmd := getEnvoy("extension build").Args(getToolchainContainerOptions()...)
 			_ = requireExecNoStderr(t, cmd)
 
 			// Verify the extension built
-			extensionWasmFile := filepath.Join(workDir, extensionWasmPath(test.Language))
+			extensionWasmFile := filepath.Join(workDir, extensionWasmPath())
 			require.FileExists(t, extensionWasmFile, `extension wasm file %s missing after running [%v]`, extensionWasmFile, cmd)
 		})
 	}

test/e2e/getenvoy_extension_examples_test.go

@@ -22,27 +22,20 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension"
-	e2e "github.com/tetratelabs/getenvoy/test/e2e/util"
 )
 
 // TestGetEnvoyExtensionExampleAdd runs the equivalent of "getenvoy extension example XXX" commands for a matrix of
-// extension.Categories and extension.Languages.
+// extension.Categories.
 //
 // "getenvoy extension example" does not use Docker. See TestMain for general notes on about the test runtime.
 func TestGetEnvoyExtensionExample(t *testing.T) {
 	const extensionName = "getenvoy_extension_example"
 
-	for _, test := range e2e.GetCategoryLanguageCombinations() {
-		test := test // pin! see https://github.com/kyoh86/scopelint for why
+	for _, category := range extension.Categories {
+		category := category // pin! see https://github.com/kyoh86/scopelint for why
 
-		t.Run(test.String(), func(t *testing.T) {
-			var extensionConfigFileName string
-			switch test.Language {
-			case extension.LanguageTinyGo:
-				extensionConfigFileName = "extension.txt"
-			default:
-				extensionConfigFileName = "extension.json"
-			}
+		t.Run(category.String(), func(t *testing.T) {
+			extensionConfigFileName := extensionConfigFileName()
 
 			workDir, removeWorkDir := requireNewTempDir(t)
 			defer removeWorkDir()
@@ -51,19 +44,19 @@ func TestGetEnvoyExtensionExample(t *testing.T) {
 			defer revertChDir()
 
 			// "getenvoy extension example XXX" commands require an extension init to succeed
-			requireExtensionInit(t, workDir, test.Category, test.Language, extensionName)
+			requireExtensionInit(t, workDir, category, extensionName)
 			defer requireExtensionClean(t, workDir)
 
 			// "getenvoy extension examples list" should start empty
-			cmd := GetEnvoy("extension examples list")
+			cmd := getEnvoy("extension examples list")
 			stderr := requireExecNoStdout(t, cmd)
 			require.Equal(t, `Extension has no example setups.
 
 Use "getenvoy extension examples add --help" for more information on how to add one.
 `, stderr, `invalid stderr running [%v]`, cmd)
 
 			// "getenvoy extension examples add" should result in stderr describing files created.
-			cmd = GetEnvoy("extension examples add")
+			cmd = getEnvoy("extension examples add")
 			stderr = requireExecNoStdout(t, cmd)
 
 			exampleFiles := []string{
@@ -90,12 +83,12 @@ Use "getenvoy extension examples add --help" for more information on how to add
 			}
 
 			// "getenvoy extension examples list" should now include an example
-			cmd = GetEnvoy("extension examples list")
+			cmd = getEnvoy("extension examples list")
 			stdout := requireExecNoStderr(t, cmd)
 			require.Equal(t, "EXAMPLE\ndefault\n", stdout, `invalid stdout running [%v]`, cmd)
 
 			// "getenvoy extension examples add" should result in stderr describing files created.
-			cmd = GetEnvoy("extension examples remove --name default")
+			cmd = getEnvoy("extension examples remove --name default")
 			stderr = requireExecNoStdout(t, cmd)
 
 			// Check stderr mentions the files removed

test/e2e/getenvoy_extension_init_test.go

@@ -24,11 +24,9 @@ import (
 	workspaces "github.com/tetratelabs/getenvoy/pkg/extension/workspace"
 	"github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension"
 	toolchains "github.com/tetratelabs/getenvoy/pkg/extension/workspace/toolchain"
-	e2e "github.com/tetratelabs/getenvoy/test/e2e/util"
 )
 
-// TestGetEnvoyExtensionInit runs the equivalent of "getenvoy extension init" for a matrix of extension.Categories and
-// extension.Languages.
+// TestGetEnvoyExtensionInit runs the equivalent of "getenvoy extension init" for a matrix of extension.Categories.
 //
 // "getenvoy extension init" does not use Docker. See TestMain for general notes on about the test runtime.
 func TestGetEnvoyExtensionInit(t *testing.T) {
@@ -37,15 +35,14 @@ func TestGetEnvoyExtensionInit(t *testing.T) {
 	type testTuple struct {
 		testName string
 		extension.Category
-		extension.Language
 		currentDirectory bool
 	}
 
 	tests := make([]testTuple, 0)
-	for _, c := range e2e.GetCategoryLanguageCombinations() {
+	for _, c := range extension.Categories {
 		tests = append(tests,
-			testTuple{c.String() + "-currentDirectory", c.Category, c.Language, true},
-			testTuple{c.String() + "-newDirectory", c.Category, c.Language, false},
+			testTuple{c.String() + "-currentDirectory", c, true},
+			testTuple{c.String() + "-newDirectory", c, false},
 		)
 	}
 
@@ -64,10 +61,10 @@ func TestGetEnvoyExtensionInit(t *testing.T) {
 			}
 
 			// "getenvoy extension init" should result in stderr describing files created.
-			cmd := GetEnvoy("extension init").
+			cmd := getEnvoy("extension init").
 				Arg(workDir).
 				Arg("--category").Arg(test.Category.String()).
-				Arg("--language").Arg(test.Language.String()).
+				Arg("--language").Arg(extensionLanguage.String()).
 				Arg("--name").Arg(extensionName)
 			stderr := requireExecNoStdout(t, cmd)
 
@@ -91,7 +88,7 @@ func TestGetEnvoyExtensionInit(t *testing.T) {
 			require.NotNil(t, workspace, `nil workspace running [%v]`, cmd)
 			require.Equal(t, extensionName, workspace.GetExtensionDescriptor().Name, `wrong extension name running [%v]`, cmd)
 			require.Equal(t, test.Category, workspace.GetExtensionDescriptor().Category, `wrong extension category running [%v]`, cmd)
-			require.Equal(t, test.Language, workspace.GetExtensionDescriptor().Language, `wrong extension language running [%v]`, cmd)
+			require.Equal(t, extensionLanguage, workspace.GetExtensionDescriptor().Language, `wrong extension extensionLanguage running [%v]`, cmd)
 
 			// Check the default toolchain is loadable
 			toolchain, err := toolchains.LoadToolchain(toolchains.Default, workspace)

test/e2e/getenvoy_extension_push_test.go

@@ -41,35 +41,27 @@ func TestGetEnvoyExtensionPush(t *testing.T) {
 	// When unspecified, we default the tag to Docker's default "latest". Note: recent tools enforce qualifying this!
 	const defaultTag = "latest"
 
-	type testTuple struct {
-		name string
-		extension.Category
-		extension.Language
-	}
-
-	// Push is not language-specific, so we don't need to test a large matrix, and doing so would slow down e2e runtime.
+	// Push is not category-specific, so we don't need to test a large matrix, and doing so would slow down e2e runtime.
 	// Instead, we choose something that executes "getenvoy extension build" quickly.
-	tests := []testTuple{
-		{"tinygo HTTP filter", extension.EnvoyHTTPFilter, extension.LanguageTinyGo},
-	}
+	categories := []extension.Category{extension.EnvoyHTTPFilter}
 
-	for _, test := range tests {
-		test := test // pin! see https://github.com/kyoh86/scopelint for why
+	for _, category := range categories {
+		category := category // pin! see https://github.com/kyoh86/scopelint for why
 
-		t.Run(test.name, func(t *testing.T) {
+		t.Run(category.String(), func(t *testing.T) {
 			workDir, removeWorkDir := requireNewTempDir(t)
 			defer removeWorkDir()
 
 			revertChDir := requireChDir(t, workDir)
 			defer revertChDir()
 
 			// push requires "get envoy extension init" and "get envoy extension build" to have succeeded
-			requireExtensionInit(t, workDir, test.Category, test.Language, extensionName)
+			requireExtensionInit(t, workDir, category, extensionName)
 			defer requireExtensionClean(t, workDir)
-			wasmBytes := requireExtensionBuild(t, test.Language, workDir)
+			wasmBytes := requireExtensionBuild(t, workDir)
 
 			// After pushing, stderr should include the registry URL and the image tag.
-			cmd := GetEnvoy("extension push").Arg(localRegistryWasmImageRef)
+			cmd := getEnvoy("extension push").Arg(localRegistryWasmImageRef)
 			stderr := requireExecNoStdout(t, cmd)
 
 			// Assemble a fully-qualified image ref as we'll pull this later

test/e2e/getenvoy_extension_run_test.go

@@ -27,37 +27,38 @@ import (
 	"github.com/tetratelabs/log"
 
 	"github.com/tetratelabs/getenvoy/pkg/common"
+	"github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension"
 	e2e "github.com/tetratelabs/getenvoy/test/e2e/util"
 	utilenvoy "github.com/tetratelabs/getenvoy/test/e2e/util/envoy"
 )
 
 const extensionName = "getenvoy_extension_run"
 const terminateTimeout = 2 * time.Minute
 
-// TestGetEnvoyExtensionRun runs the equivalent of "getenvoy extension run" for a matrix of extension.Categories and
-// extension.Languages. "getenvoy extension init" is a prerequisite, so run first.
+// TestGetEnvoyExtensionRun runs the equivalent of "getenvoy extension run" for a matrix of extension.Categories.
+// "getenvoy extension init" is a prerequisite, so run first.
 //
 // "getenvoy extension run" uses Docker. See TestMain for general notes on about the test runtime.
 func TestGetEnvoyExtensionRun(t *testing.T) {
 	debugDir, revertOriginalDebugDir := backupDebugDir(t)
 	defer revertOriginalDebugDir()
 
-	for _, test := range e2e.GetCategoryLanguageCombinations() {
-		test := test // pin! see https://github.com/kyoh86/scopelint for why
+	for _, category := range extension.Categories {
+		category := category // pin! see https://github.com/kyoh86/scopelint for why
 
-		t.Run(test.String(), func(t *testing.T) {
+		t.Run(category.String(), func(t *testing.T) {
 			workDir, removeWorkDir := requireNewTempDir(t)
 			defer removeWorkDir()
 
 			revertChDir := requireChDir(t, workDir)
 			defer revertChDir()
 
 			// run requires "get envoy extension init" to have succeeded
-			requireExtensionInit(t, workDir, test.Category, test.Language, extensionName)
+			requireExtensionInit(t, workDir, category, extensionName)
 			defer requireExtensionClean(t, workDir)
 
 			// "getenvoy extension run" only returns stdout because `docker run -t` redirects stderr to stdout.
-			cmd := GetEnvoy("extension run --envoy-options '-l trace'").Args(getBuiltinContainerOptions()...)
+			cmd := getEnvoy("extension run --envoy-options '-l trace'").Args(getToolchainContainerOptions()...)
 			_, stderr, terminate := cmd.Start(t, terminateTimeout)
 
 			// The underlying call is conditional to ensure errors that raise before we stop the server, stop it.
@@ -136,7 +137,7 @@ func TestGetEnvoyExtensionRun(t *testing.T) {
 // Typically, this will run in the default ~/.getenvoy directory, as a means to avoid re-downloads of files such as
 // .getenvoy/builds/standard/1.17.0/darwin/bin/envoy (~100MB)
 //
-// While CI usually overrides the `HOME` variable with E2E_BUILTIN_TOOLCHAIN_CONTAINER_OPTIONS, a developer may be
+// While CI usually overrides the `HOME` variable with E2E_TOOLCHAIN_CONTAINER_OPTIONS, a developer may be
 // running this on their laptop. To avoid clobbering their old debug data, backup the
 func backupDebugDir(t *testing.T) (string, func()) {
 	debugDir := filepath.Join(common.HomeDir, "debug")