Changes from statik to go:embed for example and extension templates

.github/workflows/commit.yaml

@@ -42,9 +42,6 @@ jobs:
           docker_buildx: false  # Install is flakey. When it, we can install it via docker/setup-buildx-action@v1
         timeout-minutes: 20  # fail fast if MacOS install takes too long
 
-      - name: "Init on first use"
-        run: make init
-
       - name: "Verify clean check-in"
         run: make check
 
@@ -79,9 +76,6 @@ jobs:
         with:
           go-version: '1.16.2'
 
-      - name: "Init on first use"
-        run: make init
-
       - name: "Build the `getenvoy` binary"
         run: make bin
 
@@ -115,6 +109,11 @@ jobs:
       - name: "Checkout"
         uses: actions/checkout@v2
 
+      - name: "Install Go"
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16.2'
+
       - name: "Re-use the `getenvoy` binary pre-built by the upstream job"
         uses: actions/download-artifact@v2
         with:
@@ -142,5 +141,8 @@ jobs:
         timeout-minutes: 10  # fail fast if MacOS runner becomes too slow
 
       - name: "Run e2e tests using the `getenvoy` binary built by the upstream job"
+        # chmod to restore permissions lost in actions/download-artifact@v2
         # expand E2E_TOOLCHAIN_CONTAINER_OPTIONS here to allow shell interpolation
-        run: E2E_TOOLCHAIN_CONTAINER_OPTIONS="${{ matrix.args.toolchain-container-options }}" make e2e
+        run: |
+          chmod a+x build/bin/*/*/getenvoy
+          E2E_TOOLCHAIN_CONTAINER_OPTIONS="${{ matrix.args.toolchain-container-options }}" make e2e

.github/workflows/release.yaml

@@ -96,6 +96,11 @@ jobs:
       - name: "Checkout"
         uses: actions/checkout@v2
 
+      - name: "Install Go"
+        uses: actions/setup-go@v2
+        with:
+          go-version: '1.16.2'
+
       - name: "Get tag name"
         run: | # Trim "v" prefix in the release tag
           RELEASE_TAG=${GITHUB_REF#refs/*/}

.gitignore

@@ -3,6 +3,3 @@ dist
 /getenvoy
 /build/
 .idea
-
-# code generated by `github.com/rakyll/statik`
-statik.go

.golangci.yml

@@ -134,11 +134,6 @@ issues:
         - gosec
         - lll
 
-    # Exclude lll issues for long lines with go:generate
-    - linters:
-        - lll
-      source: "^//go:generate "
-
   # Independently from option `exclude` we use default exclude patterns,
   # it can be disabled by this option. To list all
   # excluded by default patterns execute `golangci-lint run --help`.

.goreleaser.yaml

@@ -15,9 +15,6 @@
 project_name: getenvoy
 env:
   - GO111MODULE=on
-before:
-  hooks:
-    - make init
 builds:
 - binary: getenvoy
   ldflags: "-s -w -X github.com/tetratelabs/getenvoy/pkg/version.version={{.Version}}"

Makefile

@@ -66,22 +66,15 @@ GETENVOY_OUT_PATH = $(BIN_DIR)/$(1)/$(2)/getenvoy
 
 define GEN_GETENVOY_BUILD_TARGET
 .PHONY: $(call GETENVOY_OUT_PATH,$(1),$(2))
-$(call GETENVOY_OUT_PATH,$(1),$(2)): generate
+$(call GETENVOY_OUT_PATH,$(1),$(2)):
 	CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build $(GO_LD_FLAGS) -o $(call GETENVOY_OUT_PATH,$(1),$(2)) ./cmd/getenvoy/main.go
 endef
 $(foreach os,$(GOOSES),$(foreach arch,$(GOARCHS),$(eval $(call GEN_GETENVOY_BUILD_TARGET,$(os),$(arch)))))
 
-.PHONY: init
-init: generate
-
 .PHONY: deps
 deps:
 	go mod download
 
-.PHONY: generate
-generate: deps
-	go generate ./pkg/...
-
 .PHONY: build
 build: $(call GETENVOY_OUT_PATH,$(GOOS),$(GOARCH))
 
@@ -94,7 +87,7 @@ release.dryrun:
 	goreleaser release --skip-publish --snapshot --rm-dist
 
 .PHONY: test
-test: generate
+test:
 	docker-compose up -d
 	go test $(GO_TEST_OPTS) $(GO_TEST_EXTRA_OPTS) $(TEST_PKG_LIST)
 
@@ -119,7 +112,7 @@ endef
 $(foreach os,$(GOOSES),$(foreach arch,$(GOARCHS),$(eval $(call GEN_BIN_GOOS_GOARCH_TARGET,$(os),$(arch)))))
 
 .PHONY: coverage
-coverage: generate
+coverage:
 	mkdir -p "$(shell dirname "$(COVERAGE_PROFILE)")"
 	go test $(GO_COVERAGE_OPTS) $(GO_COVERAGE_EXTRA_OPTS) -coverprofile="$(COVERAGE_PROFILE)" $(COVERAGE_PKG_LIST)
 	go tool cover -html="$(COVERAGE_PROFILE)" -o "$(COVERAGE_REPORT)"
@@ -176,8 +169,7 @@ builders.pull: $(foreach lang,$(BUILDERS_LANGS), pull/builder/$(lang))
 
 LINT_OPTS ?= --timeout 5m
 .PHONY: lint
-# generate must be called while generated source is still used
-lint: generate $(GOLANGCI_LINT) $(SHFMT) $(LICENSER) .golangci.yml  ## Run the linters
+lint: $(GOLANGCI_LINT) $(SHFMT) $(LICENSER) .golangci.yml  ## Run the linters
 	@echo "--- lint ---"
 	@$(SHFMT) -d .
 	@$(LICENSER) verify -r .

RATIONALE.md

@@ -0,0 +1,67 @@
+# Notable Rationale
+
+## go:embed for embedding example and init templates
+
+This project resolves templates into working examples or extensions via `getenvoy example add` and
+`getenvoy extension init`. Input [example](data/example/init/templates) and [extension](data/extension/init/templates)
+templates are embedded in the `getenvoy` binary for user convenience.
+
+We implement embedding with a Go 1.16+ directive `//go:embed templates/*`, which presents an `fs.FS` interface of the
+directory tree. Using this requires no extra build steps. Ex. `go build -o getenvoy ./cmd/getenvoy/main.go` works.
+
+However, there are some constraints to understand. It is accepted that while imperfect, this solution is an acceptable
+alternative to a custom solution.
+
+### Embedded files must be in a sub-path
+The go source file embedding content via `go:embed` must reference paths in the current directory tree. In other words,
+paths like `../../templates` are not allowed.
+
+This means we have to move the embedded directory tree where code refers to it, or make an accessor utility for each
+directory root. We use the latter pattern, specifically here:
+* [data/example/init/templates.go](data/example/init/templates.go) 
+* [data/extension/init/templates.go](data/extension/init/templates.go)
+
+See https://pkg.go.dev/embed#hdr-Directives for more information.
+
+### Limitations of `go:embed` impact extension init templates
+`getenvoy extension init` creates a workspace directory for a given category and programming language. Some constraints
+of `go:embed` impact how these template directories are laid out, and the workarounds are file rename in basis. The
+impacts are noted below for information and future follow-up:
+
+#### `go:embed` doesn't traverse hidden directories, but Rust projects include a hidden directory 
+Our Rust examples use [Cargo](https://doc.rust-lang.org/cargo/reference/config.html) as a build tool. This stores
+configuration in a hidden directory `.cargo`. As of Go 1.16, hidden directories are not yet supported with `go:embed`.
+See https://github.com/golang/go/issues/43854
+
+#### `go:embed` stops traversing at module boundaries, and TinyGo examples look like sub-modules
+Go modules need to be built from the zip-file uploaded to the mirror. This implies `go:embed` must refer to the
+current module. https://github.com/golang/go/issues/45197 explains embedding stops traversing when it encounters a
+`go.mod` file, and there is no plan to change this.
+
+This presents a challenge with TinyGo templates, which except for parameterization like `module {{ .Extension.Name }}`,
+appear as normal go modules (ex `go.mod`) files. When `go:embed` encounters a `go.mod` file, it stops traversing. If we
+didn't work around this, no TinyGo project would end up in the embedded filesystem.
+
+The workaround is to rename the template `go.mod` to `go.mod_`, but this introduces a couple glitches:
+
+* TinyGo extension templates imports appear in the root project after running `go mod tidy`
+  * As of the writing, there is only one github.com/tetratelabs/proxy-wasm-go-sdk
+* TinyGo extension templates can't be run from their directory without temporarily renaming `go.mod_` back to `go.mod`
+  * To do this anyway, you'd need to fix any template variables like `module {{ .Extension.Name }}`
+
+### Former solution
+In the past, we embedded via [statik](https://github.com/rakyll/statik). This is a code generation solution which
+encodes files into internal variables. Entry-points use an `http.FileSystem` to access the embedded files.
+
+This solution worked well, except that it introduced build complexity. Code generation required an `init` phase in the
+`Makefile`, as well lint exclusions. This implied steps for developers to remember and CI to invoke.
+
+It also introduced maintenance and risk as the statik library stalled. This project ended up pinned to a personal fork
+in order to work around unmerged pull requests.
+```
+replace github.com/rakyll/statik => github.com/yskopets/statik v0.1.8-0.20200501213002-c2d8dcc79889
+```
+
+Go 1.16's `go:embed` has limitations of its own, but brings with it shared understanding. While we may need workarounds
+to some issues, those issues are understood by the go community. `go:embed` does not require our maintenance, nor has
+any key person risk.

data/example/init/templates.go

@@ -0,0 +1,35 @@
+// Copyright 2021 Tetrate
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package init
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// templatesFs includes only the relative path of "templates".
+//
+// See RATIONALE.md for more information on embedding
+//go:embed templates/*
+var templatesFs embed.FS
+
+// GetTemplates returns the templates directory as a filesystem
+func GetTemplates() fs.FS {
+	f, err := fs.Sub(templatesFs, "templates")
+	if err != nil {
+		panic(err) // unexpected or a typo
+	}
+	return f
+}

data/extension/init/templates.go

@@ -0,0 +1,35 @@
+// Copyright 2021 Tetrate
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package init
+
+import (
+	"embed"
+	"io/fs"
+)
+
+// templatesFs includes only the relative path of "templates".
+//
+// See RATIONALE.md for more information on embedding
+//go:embed templates/*
+var templatesFs embed.FS
+
+// GetTemplates returns the templates directory as a filesystem
+func GetTemplates() fs.FS {
+	f, err := fs.Sub(templatesFs, "templates")
+	if err != nil {
+		panic(err) // unexpected or a typo
+	}
+	return f
+}

data/extension/init/templates/rust/.gitignore

@@ -1,2 +1,4 @@
+# We rename .cargo to cargo See /RATIONALE.md for why
+.cargo
 Cargo.lock
 target/

data/extension/init/templates/tinygo/.gitignore

@@ -1 +1,4 @@
 build/
+
+# We rename go.mod to go.mod_ See /RATIONALE.md for why
+go.mod

go.mod

@@ -1,5 +1,6 @@
 module github.com/tetratelabs/getenvoy
 
+// This project uses go:embed, so requires minimally go 1.16
 go 1.16
 
 require (
@@ -16,7 +17,7 @@ require (
 	github.com/go-ole/go-ole v1.2.4 // indirect
 	github.com/golang/protobuf v1.3.5
 	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
-	github.com/manifoldco/promptui v0.0.0-00010101000000-000000000000
+	github.com/manifoldco/promptui v0.8.0
 	github.com/mattn/go-isatty v0.0.12
 	github.com/mattn/go-shellwords v1.0.10
 	github.com/mholt/archiver v3.1.1+incompatible
@@ -27,7 +28,6 @@ require (
 	github.com/opencontainers/selinux v1.8.0 // indirect
 	github.com/otiai10/copy v1.2.0
 	github.com/pkg/errors v0.9.1
-	github.com/rakyll/statik v0.0.0-00010101000000-000000000000
 	github.com/schollz/progressbar/v2 v2.13.2
 	github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
@@ -36,6 +36,8 @@ require (
 	github.com/tetratelabs/getenvoy-package v0.0.0-20190730071641-da31aed4333e
 	github.com/tetratelabs/log v0.0.0-20190710134534-eb04d1e84fb8
 	github.com/tetratelabs/multierror v1.1.0
+	// Match data/extension/init/templates/tinygo/*/default/go.mod_ See RATIONALE.md for why
+	github.com/tetratelabs/proxy-wasm-go-sdk v0.1.1
 	istio.io/api v0.0.0-20200227213531-891bf31f3c32
 	istio.io/istio v0.0.0-20200304114959-c3c353285578
 	rsc.io/letsencrypt v0.0.3 // indirect
@@ -46,7 +48,3 @@ replace github.com/Azure/go-autorest/autorest => github.com/Azure/go-autorest/au
 replace github.com/docker/docker => github.com/docker/docker v17.12.1-ce+incompatible
 
 replace github.com/hashicorp/consul => github.com/hashicorp/consul v1.3.1
-
-replace github.com/manifoldco/promptui => github.com/yskopets/promptui v0.7.1-0.20200429230902-361491009c11
-
-replace github.com/rakyll/statik => github.com/yskopets/statik v0.1.8-0.20200501213002-c2d8dcc79889

go.sum

@@ -514,6 +514,8 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
+github.com/manifoldco/promptui v0.8.0 h1:R95mMF+McvXZQ7j1g8ucVZE1gLP3Sv6j9vlF9kyRqQo=
+github.com/manifoldco/promptui v0.8.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ=
 github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
 github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
@@ -738,6 +740,7 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
@@ -747,6 +750,8 @@ github.com/tetratelabs/log v0.0.0-20190710134534-eb04d1e84fb8 h1:a7FN/XPymdzttMa
 github.com/tetratelabs/log v0.0.0-20190710134534-eb04d1e84fb8/go.mod h1:w+dEBsxcYEFg0I6whrgkMzjD8GBBQgmDq9hykB30pt8=
 github.com/tetratelabs/multierror v1.1.0 h1:cKmV/Pbf42K5wp8glxa2YIausbxIraPN8fzru9Pn1Cg=
 github.com/tetratelabs/multierror v1.1.0/go.mod h1:kH3SzI/z+FwEbV9bxQDx4GiIgE2djuyb8wiB2DaUBnY=
+github.com/tetratelabs/proxy-wasm-go-sdk v0.1.1 h1:m8O4nWCyb+8VlAxVxppobGNnC1N9CoAQBmMfRjTq/hU=
+github.com/tetratelabs/proxy-wasm-go-sdk v0.1.1/go.mod h1:y1ZQT4bQEBnR8Do4nSOzb3roczzPvcAp8UrF6NEYWNY=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -775,10 +780,6 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:
 github.com/yashtewari/glob-intersection v0.0.0-20180206001645-7af743e8ec84/go.mod h1:HptNXiXVDcJjXe9SqMd0v2FsL9f8dz4GnXgltU6q/co=
 github.com/yl2chen/cidranger v0.0.0-20180214081945-928b519e5268 h1:lkoOjizoHqOcEFsvYGE5c8Ykdijjnd0R3r1yDYHzLno=
 github.com/yl2chen/cidranger v0.0.0-20180214081945-928b519e5268/go.mod h1:mq0zhomp/G6rRTb0dvHWXRHr/2+Qgeq5hMXfJ670+i4=
-github.com/yskopets/promptui v0.7.1-0.20200429230902-361491009c11 h1:MlzMpHq1fRfH1RYzfQ7Ch7JjdGnBq/m29jJtPOExWuw=
-github.com/yskopets/promptui v0.7.1-0.20200429230902-361491009c11/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ=
-github.com/yskopets/statik v0.1.8-0.20200501213002-c2d8dcc79889 h1:f62aKW+gryXYYtNut+3b6i5n1ioXCXJWpDgA11l6Pak=
-github.com/yskopets/statik v0.1.8-0.20200501213002-c2d8dcc79889/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/yuin/gopher-lua v0.0.0-20180316054350-84ea3a3c79b3/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU=
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI=
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=

pkg/extension/example/init/registry/default.go

@@ -14,26 +14,19 @@
 
 package registry
 
-//go:generate go run github.com/rakyll/statik -p=templates -m -ns=example/init/templates -src=../../../../../data/example/init/templates -a -include=* -f
-
 import (
+	"net/http"
 	"path"
 
-	"github.com/rakyll/statik/fs"
-
-	// force execution of auto generated code
-	_ "github.com/tetratelabs/getenvoy/pkg/extension/example/init/registry/templates"
+	exampleTemplates "github.com/tetratelabs/getenvoy/data/example/init"
 	"github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension"
 )
 
+var templatesFs = exampleTemplates.GetTemplates()
+
 func newDefaultRegistry() registry {
-	fileSystem, err := fs.NewWithNamespace("example/init/templates")
-	if err != nil {
-		// must be caught by unit tests
-		panic(err)
-	}
 	return &fsRegistry{
-		fs: fileSystem,
+		fs: http.FS(templatesFs),
 		namingScheme: func(category extension.Category, example string) string {
 			return "/" + path.Join(category.String(), example)
 		},