Propagates runtime.GOOS as docker env GETENVOY_GOOS to speed up rust (#…

…158)

The current way we build extensions is to mount a volume containing a directory created by "getenvoy extension init" in a "docker run" command. This implies a "bind mount" which is slow on macOS and particularly slow in rust (cargo) builds. This topic has locked our project into an old docker script, and even then it is quite slow.

This change tries to work around the problem by sharing information about the docker host (eg the machine calling getenvoy). What it does is propagates runtime.GOOS as docker env GETENVOY_GOOS. The rust builder image looks at this and optimizes if it is macOS.

Signed-off-by: Adrian Cole <adrian@tetrate.io>

images/extension-builders/entrypoint.sh

@@ -14,25 +14,19 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -e
+set -ue
 
 SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}")"  && pwd)
-GETENVOY_WORKSPACE_DIR="${GETENVOY_WORKSPACE_DIR:-$PWD}"
 
-USAGE="usage: build [--output-file PATH]
+# Note: we are using option syntax for --output-file even if this was always required. This is for compatibility with
+# older versions of the getenvoy binary.
+USAGE="usage: build --output-file path/to/extension.wasm
    or: test
    or: clean
 
 examples:
-   # build Wasm extension (location of *.wasm file is undefined)
-   build
-
    # build Wasm extension and copy *.wasm file to a given location
    build --output-file target/extension.wasm
-
-options:
-   build:
-   --output-file PATH   Path relative to the workspace root to copy *.wasm file to
 "
 
 usage() {

images/extension-builders/rust/Dockerfile

@@ -19,9 +19,9 @@ FROM rust:1.44.1
 
 RUN rustup target add wasm32-unknown-unknown
 
-# Unset CARGO_HOME. This way we will be able to determine when a user
-# provides an override value.
+# Unset CARGO_HOME and CARGO_TARGET_DIR. This way we will know when a user overrides them.
 ENV CARGO_HOME=
+ENV CARGO_TARGET_DIR=
 
 COPY ./entrypoint.sh /usr/local/getenvoy/extension/builder/entrypoint.sh
 COPY ./rust/commands.sh /usr/local/getenvoy/extension/builder/commands.sh

images/extension-builders/rust/commands.sh

@@ -13,31 +13,75 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+set -ue
 
-# Ensure location of the build directory.
-export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-${GETENVOY_WORKSPACE_DIR}/target}"
+export CARGO_TARGET="wasm32-unknown-unknown"
 
-# Keep Cargo cache inside the build directory, unless a user explicitly
-# overrides CARGO_HOME.
-export CARGO_HOME="${CARGO_HOME:-${CARGO_TARGET_DIR}/.cache/getenvoy/extension/rust-builder/cargo}"
+# Ensure location of the build cache directory.
+# See https://doc.rust-lang.org/cargo/guide/build-cache.html
+export CARGO_TARGET_DIR=${CARGO_TARGET_DIR:-${PWD}/target}
+
+# Keep Cargo cache inside the build directory, unless a user explicitly overrides CARGO_HOME.
+# See https://doc.rust-lang.org/cargo/guide/cargo-home.html
+export CARGO_HOME=${CARGO_HOME:-${CARGO_TARGET_DIR}/.cache/getenvoy/extension/rust-builder/cargo}
+
+# This is used in macOS, where we build in temporary directories. We only cache minimum as copying is slow.
+# See https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-c
+copy_cargo_home_cache() {
+	source_dir=${1}
+	dest_dir=${2}
+
+	# See if there is anything to copy
+	dirs=""
+	for dir in bin registry/index registry/cache git/db; do
+		test -d "${source_dir}/$dir" && dirs="$dirs $dir"
+	done
+
+	# If any directory existed, copy them to the destination
+	if [ -n "$dirs" ]; then
+		mkdir -p "${dest_dir}" 2>&- || true
+		log_message "     Copying cacheable dirs $dirs from ${source_dir} to ${dest_dir}"
+		(
+			cd "${source_dir}"
+			tar -cpf - $dirs | (
+				cd "${dest_dir}"
+				tar -xpf -
+			)
+		)
+	fi
+}
 
 #########################################################################
 # Build Wasm extension and copy *.wasm file to a given location.
 # Globals:
+#   CARGO_HOME
 #   CARGO_TARGET_DIR
-#   GETENVOY_WORKSPACE_DIR
+#   GETENVOY_GOOS
 # Arguments:
 #   Path relative to the workspace root to copy *.wasm file to.
 #########################################################################
-extension_build()  {
-	local target="wasm32-unknown-unknown"
+extension_build() {
+	# Avoid slow IO problems when the Docker host is macOS and bind-mounted volumes. We do this by building into a
+	# temp directory, copying back cacheable contents later.
+	if [ "${GETENVOY_GOOS:-}" == "darwin" ]; then
+		REAL_CARGO_HOME=${CARGO_HOME}
+		export CARGO_HOME=/tmp/$$-cargo
+		copy_cargo_home_cache "${REAL_CARGO_HOME}" "${CARGO_HOME}"
+
+		export CARGO_TARGET_DIR=/tmp/$$-build
+		# We don't copy revert the updated target dir back to the original location because the copying is slower than
+		# recompiling each time.
+	fi
+
+	mkdir -p "${CARGO_HOME}" 2>&- || true
+	mkdir -p "${CARGO_TARGET_DIR}" 2>&- || true
 
-	cargo build --target "${target}"
+	cargo build --target "${CARGO_TARGET}"
 
 	local profile="debug"
 	local lib_name="extension"
 	local file_name="${lib_name}.wasm"
-	local cargo_output_file="${CARGO_TARGET_DIR}/${target}/${profile}/${file_name}"
+	local cargo_output_file="${CARGO_TARGET_DIR}/${CARGO_TARGET}/${profile}/${file_name}"
 
 	if [[ ! -f "${cargo_output_file}" ]]; then
 		error "Cargo didn't build a *.wasm file at expected location: ${cargo_output_file}.
@@ -53,22 +97,21 @@ help:  make sure Cargo workspace includes a library crate with name '${lib_name}
 "
 	fi
 
-	local destination_file="$1"
-	if [[ -n "${destination_file}" ]]; then
-		log_message "     Copying *.wasm file to '${destination_file}'"
+	local destination_file="${PWD}/$1"
+	log_message "     Copying *.wasm file to '${destination_file}'"
+	mkdir -p "$(dirname "${destination_file}")"
+	cp "${cargo_output_file}" "${destination_file}"
 
-		destination_file="${GETENVOY_WORKSPACE_DIR}/${destination_file}"
-		local tmp_file="${destination_file}.tmp"
-		mkdir -p "$(dirname "${tmp_file}")"
-		cp "${cargo_output_file}" "${tmp_file}"
-		mv "${tmp_file}" "${destination_file}"
+	if [ "${GETENVOY_GOOS:-}" == "darwin" ]; then
+		copy_cargo_home_cache "${CARGO_HOME}" "${REAL_CARGO_HOME}"
+		rm -rf /tmp/$$*
 	fi
 }
 
-extension_test()  {
+extension_test() {
 	cargo test
 }
 
-extension_clean()  {
+extension_clean() {
 	cargo clean
 }

images/extension-builders/tinygo/commands.sh

@@ -13,6 +13,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+set -ue
 
 extension_build()  {
 	tinygo build -o "$1" -scheduler=none -target wasi main.go

pkg/cmd/extension/build/cmd_test.go

@@ -17,6 +17,7 @@ package build_test
 import (
 	"fmt"
 	"os/user"
+	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -105,8 +106,8 @@ func TestGetEnvoyExtensionBuild(t *testing.T) {
 	err := cmdutil.Execute(c)
 
 	// We expect docker to run from the correct path, as the current user and mount a volume for the correct workspace.
-	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s:%s --rm -t -v %s:/source -w /source --init getenvoy/extension-rust-builder:latest build --output-file target/getenvoy/extension.wasm",
-		dockerDir, expectedUser.Uid, expectedUser.Gid, workspaceDir)
+	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s:%s --rm -e GETENVOY_GOOS=%s -t -v %s:/source -w /source --init getenvoy/extension-rust-builder:latest build --output-file target/getenvoy/extension.wasm",
+		dockerDir, expectedUser.Uid, expectedUser.Gid, runtime.GOOS, workspaceDir)
 
 	// Verify the command invoked, passing the correct default commandline
 	require.NoError(t, err, `expected no error running [%v]`, c)
@@ -161,8 +162,8 @@ func TestGetEnvoyExtensionBuildFail(t *testing.T) {
 	err := cmdutil.Execute(c)
 
 	// We expect the exit instruction to have gotten to the fake docker script, along with the default options.
-	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s:%s --rm -t -v %s:/source -w /source --init %s getenvoy/extension-rust-builder:latest build --output-file target/getenvoy/extension.wasm",
-		dockerDir, expectedUser.Uid, expectedUser.Gid, workspaceDir, toolchainOptions)
+	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s:%s --rm -e GETENVOY_GOOS=%s -t -v %s:/source -w /source --init %s getenvoy/extension-rust-builder:latest build --output-file target/getenvoy/extension.wasm",
+		dockerDir, expectedUser.Uid, expectedUser.Gid, runtime.GOOS, workspaceDir, toolchainOptions)
 
 	// Verify the command failed with the expected error.
 	expectedErr := fmt.Sprintf(`failed to build Envoy extension using "default" toolchain: failed to execute an external command "%s": exit status 3`, expectedDockerExec)

pkg/cmd/extension/clean/cmd_test.go

@@ -17,6 +17,7 @@ package clean_test
 import (
 	"fmt"
 	"os/user"
+	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -105,8 +106,8 @@ func TestGetEnvoyExtensionClean(t *testing.T) {
 	err := cmdutil.Execute(c)
 
 	// We expect docker to run from the correct path, as the current user and mount a volume for the correct workspace.
-	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s:%s --rm -t -v %s:/source -w /source --init getenvoy/extension-rust-builder:latest clean",
-		dockerDir, expectedUser.Uid, expectedUser.Gid, workspaceDir)
+	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s:%s --rm -e GETENVOY_GOOS=%s -t -v %s:/source -w /source --init getenvoy/extension-rust-builder:latest clean",
+		dockerDir, expectedUser.Uid, expectedUser.Gid, runtime.GOOS, workspaceDir)
 
 	// Verify the command invoked, passing the correct default commandline
 	require.NoError(t, err, `expected no error running [%v]`, c)
@@ -161,8 +162,8 @@ func TestGetEnvoyExtensionCleanFail(t *testing.T) {
 	err := cmdutil.Execute(c)
 
 	// We expect the exit instruction to have gotten to the fake docker script, along with the default options.
-	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s:%s --rm -t -v %s:/source -w /source --init %s getenvoy/extension-rust-builder:latest clean",
-		dockerDir, expectedUser.Uid, expectedUser.Gid, workspaceDir, toolchainOptions)
+	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s:%s --rm -e GETENVOY_GOOS=%s -t -v %s:/source -w /source --init %s getenvoy/extension-rust-builder:latest clean",
+		dockerDir, expectedUser.Uid, expectedUser.Gid, runtime.GOOS, workspaceDir, toolchainOptions)
 
 	// Verify the command failed with the expected error.
 	expectedErr := fmt.Sprintf(`failed to clean build directory of Envoy extension using "default" toolchain: failed to execute an external command "%s": exit status 3`, expectedDockerExec)

pkg/cmd/extension/run/cmd_test.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"os/user"
 	"path/filepath"
+	"runtime"
 	"testing"
 
 	"github.com/Masterminds/semver"
@@ -169,11 +170,11 @@ func TestGetEnvoyExtensionRun(t *testing.T) {
 	envoyWd := cmd.ParseEnvoyWorkDirectory(stdout)
 
 	// We expect docker to build from the correct path, as the current user and mount a volume for the correct workspace.
-	expectedStdout := fmt.Sprintf(`%s/docker run -u %s --rm -t -v %s:/source -w /source --init getenvoy/extension-rust-builder:latest build --output-file target/getenvoy/extension.wasm
+	expectedStdout := fmt.Sprintf(`%s/docker run -u %s --rm -e GETENVOY_GOOS=%s -t -v %s:/source -w /source --init getenvoy/extension-rust-builder:latest build --output-file target/getenvoy/extension.wasm
 envoy pwd: %s
 envoy bin: %s
 envoy args: -c %s/envoy.tmpl.yaml`,
-		config.dockerDir, config.expectedUidGid, config.workspaceDir, envoyWd, envoyBin, envoyWd)
+		config.dockerDir, config.expectedUidGid, runtime.GOOS, config.workspaceDir, envoyWd, envoyBin, envoyWd)
 	require.Equal(t, expectedStdout+"\n", stdout.String(), `expected stdout running [%v]`, c)
 	require.Equal(t, "docker stderr\nenvoy stderr\n", stderr.String(), `expected stderr running [%v]`, c)
 
@@ -199,8 +200,8 @@ func TestGetEnvoyExtensionRunDockerFail(t *testing.T) {
 	err := cmdutil.Execute(c)
 
 	// We expect the exit instruction to have gotten to the fake docker script, along with the default options.
-	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s --rm -t -v %s:/source -w /source --init %s getenvoy/extension-rust-builder:latest build --output-file target/getenvoy/extension.wasm",
-		config.dockerDir, config.expectedUidGid, config.workspaceDir, toolchainOptions)
+	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s --rm -e GETENVOY_GOOS=%s -t -v %s:/source -w /source --init %s getenvoy/extension-rust-builder:latest build --output-file target/getenvoy/extension.wasm",
+		config.dockerDir, config.expectedUidGid, runtime.GOOS, config.workspaceDir, toolchainOptions)
 
 	// Verify the command failed with the expected error.
 	expectedErr := fmt.Sprintf(`failed to build Envoy extension using "default" toolchain: failed to execute an external command "%s": exit status 3`, expectedDockerExec)

pkg/cmd/extension/test/cmd_test.go

@@ -17,6 +17,7 @@ package test_test
 import (
 	"fmt"
 	"os/user"
+	"runtime"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -105,8 +106,8 @@ func TestGetEnvoyExtensionTest(t *testing.T) {
 	err := cmdutil.Execute(c)
 
 	// We expect docker to run from the correct path, as the current user and mount a volume for the correct workspace.
-	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s:%s --rm -t -v %s:/source -w /source --init getenvoy/extension-rust-builder:latest build --output-file target/getenvoy/extension.wasm",
-		dockerDir, expectedUser.Uid, expectedUser.Gid, workspaceDir)
+	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s:%s --rm -e GETENVOY_GOOS=%s -t -v %s:/source -w /source --init getenvoy/extension-rust-builder:latest build --output-file target/getenvoy/extension.wasm",
+		dockerDir, expectedUser.Uid, expectedUser.Gid, runtime.GOOS, workspaceDir)
 
 	// Verify the command invoked, passing the correct default commandline
 	require.NoError(t, err, `expected no error running [%v]`, c)
@@ -161,8 +162,8 @@ func TestGetEnvoyExtensionTestFail(t *testing.T) {
 	err := cmdutil.Execute(c)
 
 	// We expect the exit instruction to have gotten to the fake docker script, along with the default options.
-	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s:%s --rm -t -v %s:/source -w /source --init %s getenvoy/extension-rust-builder:latest test",
-		dockerDir, expectedUser.Uid, expectedUser.Gid, workspaceDir, toolchainOptions)
+	expectedDockerExec := fmt.Sprintf("%s/docker run -u %s:%s --rm -e GETENVOY_GOOS=%s -t -v %s:/source -w /source --init %s getenvoy/extension-rust-builder:latest test",
+		dockerDir, expectedUser.Uid, expectedUser.Gid, runtime.GOOS, workspaceDir, toolchainOptions)
 
 	// Verify the command failed with the expected error.
 	expectedErr := fmt.Sprintf(`failed to unit test Envoy extension using "default" toolchain: failed to execute an external command "%s": exit status 3`, expectedDockerExec)

pkg/extension/workspace/toolchain/builtin/toolchain.go

@@ -19,6 +19,7 @@ import (
 	"os/exec"
 	"os/user"
 	"path/filepath"
+	"runtime"
 
 	config "github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/toolchain/builtin"
 	"github.com/tetratelabs/getenvoy/pkg/extension/workspace/model"
@@ -97,6 +98,7 @@ func (t *builtin) dockerCliArgs(container *config.ContainerConfig) (executil.Arg
 		"run",
 		"-u", fmt.Sprintf("%s:%s", u.Uid, u.Gid), // to get proper ownership on files created by the container
 		"--rm",
+		"-e", "GETENVOY_GOOS=" + runtime.GOOS, // Allows builder images to act based on execution env
 		"-t", // to get interactive/colored output out of container
 		"-v", fmt.Sprintf("%s:%s", t.workspace.GetDir().GetRootDir(), "/source"),
 		"-w", "/source",