Skip to content

Commit

Permalink
Changes from statik to go:embed for examples serving
Browse files Browse the repository at this point in the history
This reduces build complexity by eliminating a generation step with go:embed. This implicitly reduces tech debt even more because not only were we using a stalled project, statik, but also a fork of it.

go:embed is not perfect, as it disallows the file name go.mod. The workaround is to rename it to go.mod_ per golang/go#45197. While this is imperfect an if-statement is a far better punch than a dependency on a fork of a stalled project which also requires a codegen step.

Signed-off-by: Adrian Cole <adrian@tetrate.io>
  • Loading branch information
Adrian Cole committed Apr 6, 2021
1 parent c58265b commit 12af57c
Show file tree
Hide file tree
Showing 21 changed files with 116 additions and 68 deletions.
11 changes: 5 additions & 6 deletions .github/workflows/commit.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,6 @@ jobs:
docker_buildx: false # Install is flakey. When it, we can install it via docker/setup-buildx-action@v1
timeout-minutes: 20 # fail fast if MacOS install takes too long

- name: "Init on first use"
run: make init

- name: "Verify clean check-in"
run: make check

Expand Down Expand Up @@ -79,9 +76,6 @@ jobs:
with:
go-version: '1.16.2'

- name: "Init on first use"
run: make init

- name: "Build the `getenvoy` binary"
run: make bin

Expand Down Expand Up @@ -115,6 +109,11 @@ jobs:
- name: "Checkout"
uses: actions/checkout@v2

- name: "Install Go"
uses: actions/setup-go@v2
with:
go-version: '1.16.2'

- name: "Re-use the `getenvoy` binary pre-built by the upstream job"
uses: actions/download-artifact@v2
with:
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,11 @@ jobs:
- name: "Checkout"
uses: actions/checkout@v2

- name: "Install Go"
uses: actions/setup-go@v2
with:
go-version: '1.16.2'

- name: "Get tag name"
run: | # Trim "v" prefix in the release tag
RELEASE_TAG=${GITHUB_REF#refs/*/}
Expand Down
3 changes: 0 additions & 3 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,3 @@ dist
/getenvoy
/build/
.idea

# code generated by `github.com/rakyll/statik`
statik.go
5 changes: 0 additions & 5 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -134,11 +134,6 @@ issues:
- gosec
- lll

# Exclude lll issues for long lines with go:generate
- linters:
- lll
source: "^//go:generate "

# Independently from option `exclude` we use default exclude patterns,
# it can be disabled by this option. To list all
# excluded by default patterns execute `golangci-lint run --help`.
Expand Down
3 changes: 0 additions & 3 deletions .goreleaser.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,6 @@
project_name: getenvoy
env:
- GO111MODULE=on
before:
hooks:
- make init
builds:
- binary: getenvoy
ldflags: "-s -w -X github.com/tetratelabs/getenvoy/pkg/version.version={{.Version}}"
Expand Down
16 changes: 4 additions & 12 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -66,22 +66,15 @@ GETENVOY_OUT_PATH = $(BIN_DIR)/$(1)/$(2)/getenvoy

define GEN_GETENVOY_BUILD_TARGET
.PHONY: $(call GETENVOY_OUT_PATH,$(1),$(2))
$(call GETENVOY_OUT_PATH,$(1),$(2)): generate
$(call GETENVOY_OUT_PATH,$(1),$(2)):
CGO_ENABLED=0 GOOS=$(1) GOARCH=$(2) go build $(GO_LD_FLAGS) -o $(call GETENVOY_OUT_PATH,$(1),$(2)) ./cmd/getenvoy/main.go
endef
$(foreach os,$(GOOSES),$(foreach arch,$(GOARCHS),$(eval $(call GEN_GETENVOY_BUILD_TARGET,$(os),$(arch)))))

.PHONY: init
init: generate

.PHONY: deps
deps:
go mod download

.PHONY: generate
generate: deps
go generate ./pkg/...

.PHONY: build
build: $(call GETENVOY_OUT_PATH,$(GOOS),$(GOARCH))

Expand All @@ -94,7 +87,7 @@ release.dryrun:
goreleaser release --skip-publish --snapshot --rm-dist

.PHONY: test
test: generate
test:
docker-compose up -d
go test $(GO_TEST_OPTS) $(GO_TEST_EXTRA_OPTS) $(TEST_PKG_LIST)

Expand All @@ -119,7 +112,7 @@ endef
$(foreach os,$(GOOSES),$(foreach arch,$(GOARCHS),$(eval $(call GEN_BIN_GOOS_GOARCH_TARGET,$(os),$(arch)))))

.PHONY: coverage
coverage: generate
coverage:
mkdir -p "$(shell dirname "$(COVERAGE_PROFILE)")"
go test $(GO_COVERAGE_OPTS) $(GO_COVERAGE_EXTRA_OPTS) -coverprofile="$(COVERAGE_PROFILE)" $(COVERAGE_PKG_LIST)
go tool cover -html="$(COVERAGE_PROFILE)" -o "$(COVERAGE_REPORT)"
Expand Down Expand Up @@ -176,8 +169,7 @@ builders.pull: $(foreach lang,$(BUILDERS_LANGS), pull/builder/$(lang))

LINT_OPTS ?= --timeout 5m
.PHONY: lint
# generate must be called while generated source is still used
lint: generate $(GOLANGCI_LINT) $(SHFMT) $(LICENSER) .golangci.yml ## Run the linters
lint: $(GOLANGCI_LINT) $(SHFMT) $(LICENSER) .golangci.yml ## Run the linters
@echo "--- lint ---"
@$(SHFMT) -d .
@$(LICENSER) verify -r .
Expand Down
36 changes: 36 additions & 0 deletions data/example/init/templates.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright 2021 Tetrate
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package init

import (
"embed"
"io/fs"
)

// templatesFs includes only the relative path of "templates".
//
// Assets must be in this directory because go:embed doesn't support navigation outside (ex. ../)
// See https://pkg.go.dev/embed#hdr-Directives
//go:embed templates/*
var templatesFs embed.FS

// GetTemplates returns the templates directory as a filesystem
func GetTemplates() fs.FS {
f, err := fs.Sub(templatesFs, "templates")
if err != nil {
panic(err) // unexpected or a typo
}
return f
}
36 changes: 36 additions & 0 deletions data/extension/init/templates.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright 2021 Tetrate
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package init

import (
"embed"
"io/fs"
)

// templatesFs includes only the relative path of "templates".
//
// Assets must be in this directory because go:embed doesn't support navigation outside (ex. ../)
// See https://pkg.go.dev/embed#hdr-Directives
//go:embed templates/*
var templatesFs embed.FS

// GetTemplates returns the templates directory as a filesystem
func GetTemplates() fs.FS {
f, err := fs.Sub(templatesFs, "templates")
if err != nil {
panic(err) // unexpected or a typo
}
return f
}
2 changes: 2 additions & 0 deletions data/extension/init/templates/rust/.gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
# We rename .cargo to cargo because go:embed doesn't recurse hidden directories
.cargo
Cargo.lock
target/
3 changes: 3 additions & 0 deletions data/extension/init/templates/tinygo/.gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,4 @@
build/

# We rename go.mod to go.mod_ to workaround https://github.com/golang/go/issues/45197
go.mod
9 changes: 3 additions & 6 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
module github.com/tetratelabs/getenvoy

// This project uses go:embed, so requires minimally go 1.16
go 1.16

require (
Expand All @@ -16,7 +17,7 @@ require (
github.com/go-ole/go-ole v1.2.4 // indirect
github.com/golang/protobuf v1.3.5
github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
github.com/manifoldco/promptui v0.0.0-00010101000000-000000000000
github.com/manifoldco/promptui v0.8.0
github.com/mattn/go-isatty v0.0.12
github.com/mattn/go-shellwords v1.0.10
github.com/mholt/archiver v3.1.1+incompatible
Expand All @@ -27,7 +28,6 @@ require (
github.com/opencontainers/selinux v1.8.0 // indirect
github.com/otiai10/copy v1.2.0
github.com/pkg/errors v0.9.1
github.com/rakyll/statik v0.0.0-00010101000000-000000000000
github.com/schollz/progressbar/v2 v2.13.2
github.com/shirou/gopsutil v0.0.0-20190731134726-d80c43f9c984
github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749
Expand All @@ -36,6 +36,7 @@ require (
github.com/tetratelabs/getenvoy-package v0.0.0-20190730071641-da31aed4333e
github.com/tetratelabs/log v0.0.0-20190710134534-eb04d1e84fb8
github.com/tetratelabs/multierror v1.1.0
github.com/tetratelabs/proxy-wasm-go-sdk v0.12.0
istio.io/api v0.0.0-20200227213531-891bf31f3c32
istio.io/istio v0.0.0-20200304114959-c3c353285578
rsc.io/letsencrypt v0.0.3 // indirect
Expand All @@ -46,7 +47,3 @@ replace github.com/Azure/go-autorest/autorest => github.com/Azure/go-autorest/au
replace github.com/docker/docker => github.com/docker/docker v17.12.1-ce+incompatible

replace github.com/hashicorp/consul => github.com/hashicorp/consul v1.3.1

replace github.com/manifoldco/promptui => github.com/yskopets/promptui v0.7.1-0.20200429230902-361491009c11

replace github.com/rakyll/statik => github.com/yskopets/statik v0.1.8-0.20200501213002-c2d8dcc79889
9 changes: 5 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -514,6 +514,8 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN
github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs=
github.com/manifoldco/promptui v0.8.0 h1:R95mMF+McvXZQ7j1g8ucVZE1gLP3Sv6j9vlF9kyRqQo=
github.com/manifoldco/promptui v0.8.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ=
github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
Expand Down Expand Up @@ -738,6 +740,7 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
Expand All @@ -747,6 +750,8 @@ github.com/tetratelabs/log v0.0.0-20190710134534-eb04d1e84fb8 h1:a7FN/XPymdzttMa
github.com/tetratelabs/log v0.0.0-20190710134534-eb04d1e84fb8/go.mod h1:w+dEBsxcYEFg0I6whrgkMzjD8GBBQgmDq9hykB30pt8=
github.com/tetratelabs/multierror v1.1.0 h1:cKmV/Pbf42K5wp8glxa2YIausbxIraPN8fzru9Pn1Cg=
github.com/tetratelabs/multierror v1.1.0/go.mod h1:kH3SzI/z+FwEbV9bxQDx4GiIgE2djuyb8wiB2DaUBnY=
github.com/tetratelabs/proxy-wasm-go-sdk v0.12.0 h1:n2foXJoViPNPz5Jr3bHMc8a4WXBuXwQ3bKJYpfXZUPE=
github.com/tetratelabs/proxy-wasm-go-sdk v0.12.0/go.mod h1:y1ZQT4bQEBnR8Do4nSOzb3roczzPvcAp8UrF6NEYWNY=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
Expand Down Expand Up @@ -775,10 +780,6 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:
github.com/yashtewari/glob-intersection v0.0.0-20180206001645-7af743e8ec84/go.mod h1:HptNXiXVDcJjXe9SqMd0v2FsL9f8dz4GnXgltU6q/co=
github.com/yl2chen/cidranger v0.0.0-20180214081945-928b519e5268 h1:lkoOjizoHqOcEFsvYGE5c8Ykdijjnd0R3r1yDYHzLno=
github.com/yl2chen/cidranger v0.0.0-20180214081945-928b519e5268/go.mod h1:mq0zhomp/G6rRTb0dvHWXRHr/2+Qgeq5hMXfJ670+i4=
github.com/yskopets/promptui v0.7.1-0.20200429230902-361491009c11 h1:MlzMpHq1fRfH1RYzfQ7Ch7JjdGnBq/m29jJtPOExWuw=
github.com/yskopets/promptui v0.7.1-0.20200429230902-361491009c11/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ=
github.com/yskopets/statik v0.1.8-0.20200501213002-c2d8dcc79889 h1:f62aKW+gryXYYtNut+3b6i5n1ioXCXJWpDgA11l6Pak=
github.com/yskopets/statik v0.1.8-0.20200501213002-c2d8dcc79889/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
github.com/yuin/gopher-lua v0.0.0-20180316054350-84ea3a3c79b3/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU=
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI=
github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
Expand Down
17 changes: 5 additions & 12 deletions pkg/extension/example/init/registry/default.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,26 +14,19 @@

package registry

//go:generate go run github.com/rakyll/statik -p=templates -m -ns=example/init/templates -src=../../../../../data/example/init/templates -a -include=* -f

import (
"net/http"
"path"

"github.com/rakyll/statik/fs"

// force execution of auto generated code
_ "github.com/tetratelabs/getenvoy/pkg/extension/example/init/registry/templates"
exampleTemplates "github.com/tetratelabs/getenvoy/data/example/init"
"github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension"
)

var templatesFs = exampleTemplates.GetTemplates()

func newDefaultRegistry() registry {
fileSystem, err := fs.NewWithNamespace("example/init/templates")
if err != nil {
// must be caught by unit tests
panic(err)
}
return &fsRegistry{
fs: fileSystem,
fs: http.FS(templatesFs),
namingScheme: func(category extension.Category, example string) string {
return "/" + path.Join(category.String(), example)
},
Expand Down
11 changes: 7 additions & 4 deletions pkg/extension/init/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,14 @@ func (s *scaffolder) walk(sourceDirName, destinationDirName string) (errs error)

func (s *scaffolder) visit(sourceDirName, destinationDirName string, sourceFileInfo os.FileInfo) (errs error) {
baseOutputFileName := sourceFileInfo.Name()
// We rename go.mod to go.mod_ to workaround https://github.com/golang/go/issues/45197
if baseOutputFileName == "go.mod_" {
baseOutputFileName = "go.mod" // rename workaround for https://github.com/golang/go/issues/45197
// This works around go:embed limitations via renaming per https://github.com/golang/go/issues/45197
// This logic should be moved to a decorating http.FileSystem when one exists.
if baseOutputFileName == "go.mod_" { // go.mod is a forbidden name
baseOutputFileName = "go.mod"
}
if destinationDirName == "cargo" { // go:embed doesn't recurse hidden directories
destinationDirName = ".cargo"
}

relOutputFileName := filepath.Join(destinationDirName, baseOutputFileName)
outputFileName := filepath.Join(s.opts.OutputDir, relOutputFileName)
if err := osutil.EnsureDirExists(filepath.Dir(outputFileName)); err != nil {
Expand Down
18 changes: 5 additions & 13 deletions pkg/extension/init/source.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,13 @@

package init

//go:generate go run github.com/rakyll/statik -p=templates -m -ns=extension/init/templates -src=../../../data/extension/init/templates -a -include=* -f

import (
"net/http"
"path"
"sync"

"github.com/rakyll/statik/fs"

extensionTemplates "github.com/tetratelabs/getenvoy/data/extension/init"
"github.com/tetratelabs/getenvoy/pkg/extension/workspace/config/extension"

// force execution of auto generated code
_ "github.com/tetratelabs/getenvoy/pkg/extension/init/templates"
)

var (
Expand All @@ -34,17 +29,14 @@ var (
templatesOnce sync.Once
)

var templatesFs = extensionTemplates.GetTemplates()

// getTemplateSource returns a source of extension templates.
func getTemplateSource() templateSource {
// do lazy init to avoid zip decompression unless absolutely necessary
templatesOnce.Do(func() {
fileSystem, err := fs.NewWithNamespace("extension/init/templates")
if err != nil {
// must be caught by unit tests
panic(err)
}
templates = &fsTemplateSource{
fs: fileSystem,
fs: http.FS(templatesFs),
namingScheme: func(language extension.Language, category extension.Category, template string) string {
return "/" + path.Join(language.String(), category.String(), template)
},
Expand Down

0 comments on commit 12af57c

Please sign in to comment.