From 15c9fae4919b60bb8da1b5e9a2961cc0df626488 Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Mon, 28 Sep 2020 04:39:16 +0900 Subject: [PATCH] feat: Embed local imports into binary Ref #28 --- .gitignore | 1 + Makefile | 15 +- .../import-multi/baz/redundant/.gitkeep | 0 examples/advanced/import-multi/export.sh | 18 ++ go.mod | 3 +- go.sum | 3 + pkg/app/app_shim.go | 126 ++++++++++--- pkg/app/export.go | 7 + pkg/app/load.go | 21 ++- pkg/conf/load.go | 11 +- pkg/fs/fs.go | 165 ++++++++++++++++++ 11 files changed, 331 insertions(+), 39 deletions(-) create mode 100644 examples/advanced/import-multi/baz/redundant/.gitkeep create mode 100755 examples/advanced/import-multi/export.sh create mode 100644 pkg/app/export.go create mode 100644 pkg/fs/fs.go diff --git a/.gitignore b/.gitignore index 5dd4159..665fd82 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ variant bin/golangci-lint bin/goimports \#*# +.variant2 diff --git a/Makefile b/Makefile index 816a61c..ee92342 100644 --- a/Makefile +++ b/Makefile @@ -27,14 +27,23 @@ lint: bin/golangci-lint --disable gomnd,funlen,prealloc,gocritic,lll,gocognit .PHONY: smoke +smoke: export GOBIN=$(shell pwd)/tools smoke: build + go get github.com/rakyll/statik + make build - ./variant export go examples/simple build/simple - go build -o build/simple/simple ./build/simple + rm -rf build/simple + PATH=${PATH}:$(GOBIN) ./variant export go examples/simple build/simple + cd build/simple; go build -o simple ./ build/simple/simple -h | tee smoke.log grep "Namespace to interact with" smoke.log rm build/simple/simple - ./variant export binary examples/simple build/simple + PATH=${PATH}:$(GOBIN) ./variant export binary examples/simple build/simple build/simple/simple -h | tee smoke2.log grep "Namespace to interact with" smoke2.log + + rm -rf build/import-multi + VARIANT_BUILD_VER=v0.0.0 VARIANT_BUILD_REPLACE=$(shell pwd) PATH=${PATH}:$(GOBIN) ./variant export binary examples/advanced/import-multi build/import-multi + build/import-multi foo baz HELLO > build/import-multi.log + bash -c 'diff <(echo HELLO) <(cat build/import-multi.log)' diff --git a/examples/advanced/import-multi/baz/redundant/.gitkeep b/examples/advanced/import-multi/baz/redundant/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/examples/advanced/import-multi/export.sh b/examples/advanced/import-multi/export.sh new file mode 100755 index 0000000..d4ee0f4 --- /dev/null +++ b/examples/advanced/import-multi/export.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +PROJECT_ROOT=../../.. +VARIANT=${PROJECT_ROOT}/variant + +export VARIANT_BUILD_VER=v0.33.3 +export VARIANT_BUILD_REPLACE=$(pwd)/${PROJECT_ROOT} + +rm -rf ../exported +rm -rf ../compiled + +(cd ${PROJECT_ROOT}; make build) +${VARIANT} export go ../import-multi ../exported +${VARIANT} export binary ../import-multi ../compiled + +${VARIANT} run foo baz HELLO1 + +(cd ..; ./compiled foo baz HELLO2) diff --git a/go.mod b/go.mod index 87b8134..98971d0 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/mattn/go-isatty v0.0.12 github.com/nlopes/slack v0.6.0 github.com/pkg/errors v0.9.1 + github.com/rakyll/statik v0.1.7 github.com/rs/xid v1.2.1 github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 // indirect @@ -38,7 +39,7 @@ require ( github.com/zclconf/go-cty-yaml v1.0.1 golang.org/x/crypto v0.0.0-20200214034016-1d94cc7ab1c6 // indirect golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a - golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 + golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4 // indirect gopkg.in/go-playground/validator.v9 v9.31.0 // indirect gopkg.in/yaml.v2 v2.2.8 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c diff --git a/go.sum b/go.sum index 4460bbc..0685fa8 100644 --- a/go.sum +++ b/go.sum @@ -902,6 +902,8 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= +github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= +github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1362,6 +1364,7 @@ google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/ google.golang.org/api v0.20.0 h1:jz2KixHX7EcCPiQrySzPdnYT7DbINAypCqKZ1Z7GM40= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.21.0 h1:zS+Q/CJJnVlXpXQVIz+lH0ZT2lBuT2ac7XD8Y/3w6hY= +google.golang.org/api v0.32.0 h1:Le77IccnTqEa8ryp9wIpX5W3zYm7Gf9LhOp9PHcwFts= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/pkg/app/app_shim.go b/pkg/app/app_shim.go index f736029..d9e43fa 100644 --- a/pkg/app/app_shim.go +++ b/pkg/app/app_shim.go @@ -3,11 +3,14 @@ package app import ( "bytes" "fmt" + "io" "io/ioutil" "os" "path/filepath" "strings" + "github.com/mumoshu/variant2/pkg/fs" + "github.com/hashicorp/hcl/v2" "github.com/mumoshu/variant2/pkg/conf" @@ -38,7 +41,7 @@ func (app *App) ExportBinary(srcDir, dstFile string) error { _, err = app.execCmd( Command{ Name: "sh", - Args: []string{"-c", fmt.Sprintf("cd %s; go mod init %s && go build -o %s %s", tmpDir, filepath.Base(srcDir), absDstFile, tmpDir)}, + Args: []string{"-c", fmt.Sprintf("cd %s; go build -o %s %s", tmpDir, absDstFile, tmpDir)}, Env: map[string]string{}, }, true, @@ -52,22 +55,12 @@ func (app *App) ExportGo(srcDir, dstDir string) error { return err } - fs, err := findVariantFiles(srcDir) - if err != nil { - return err - } - - srcs, err := loadFiles(fs...) - if err != nil { - return err - } - - files, _, err := newConfigFromSources(srcs) + a, err := New(FromDir(srcDir)) if err != nil { return err } - merged, err := merge(files) + merged, err := merge(a.Files) if err != nil { return err } @@ -83,6 +76,7 @@ import ( "strings" variant "github.com/mumoshu/variant2" + _ "${MODULE_NAME}/statik" ) func main() { @@ -116,6 +110,11 @@ func main() { } `, "`"+strings.Replace(string(merged)+"\n", "`", backquote, -1)+"`")) + moduleName := app.moduleName(srcDir) + + replaced := strings.ReplaceAll(string(code), "${MODULE_NAME}", moduleName) + code = []byte(replaced) + if err := os.MkdirAll(dstDir, 0755); err != nil { return err } @@ -126,25 +125,108 @@ func main() { return err } - return nil -} + walkErr := filepath.Walk(srcDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("walking into %s: %w", path, err) + } -func (app *App) ExportShim(srcDir, dstDir string) error { - if err := os.MkdirAll(dstDir, 0755); err != nil { - return err + rel, err := filepath.Rel(srcDir, path) + if err != nil { + return fmt.Errorf("computing path of %s relative to %s: %w", path, srcDir, err) + } + + abs := filepath.Join(dstDir, fs.VendorPrefix, rel) + + if info.IsDir() { + return os.MkdirAll(abs, 0755) + } + + return copyFile(path, abs) + }) + if walkErr != nil { + return fmt.Errorf("copying files from %s: %w", srcDir, walkErr) } - fs, err := findVariantFiles(srcDir) + _, err = app.execCmd( + Command{ + Name: "sh", + Args: []string{"-c", fmt.Sprintf("cd %s; go mod init %s && go get github.com/rakyll/statik && statik -src=%s", dstDir, moduleName, fs.VendorPrefix)}, + Env: map[string]string{}, + }, + true, + ) if err != nil { return err } - srcs, err := loadFiles(fs...) + variantVer := os.Getenv("VARIANT_BUILD_VER") + if variantVer != "" { + _, err = app.execCmd( + Command{ + Name: "sh", + Args: []string{"-c", fmt.Sprintf("cd %s; go mod edit -require=github.com/mumoshu/variant2@%s", dstDir, variantVer)}, + Env: map[string]string{}, + }, + true, + ) + if err != nil { + return err + } + } + + variantReplace := os.Getenv("VARIANT_BUILD_REPLACE") + if variantReplace != "" { + _, err = app.execCmd( + Command{ + Name: "sh", + Args: []string{"-c", fmt.Sprintf("cd %s; go mod edit -replace github.com/mumoshu/variant2@%s=%s", dstDir, variantVer, variantReplace)}, + Env: map[string]string{}, + }, + true, + ) + if err != nil { + return err + } + } + + return nil +} + +func copyFile(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + + out, err := os.Create(dst) if err != nil { + return + } + + defer func() { + cerr := out.Close() + + if err == nil { + err = cerr + } + }() + + if _, err = io.Copy(out, in); err != nil { + return + } + + err = out.Sync() + + return +} + +func (app *App) ExportShim(srcDir, dstDir string) error { + if err := os.MkdirAll(dstDir, 0755); err != nil { return err } - files, _, err := newConfigFromSources(srcs) + a, err := New(FromDir(srcDir)) if err != nil { return err } @@ -156,7 +238,7 @@ func (app *App) ExportShim(srcDir, dstDir string) error { binName = "variant" } - return exportWithShim(binName, files, dstDir) + return exportWithShim(binName, a.Files, dstDir) } func merge(files map[string]*hcl.File) ([]byte, error) { diff --git a/pkg/app/export.go b/pkg/app/export.go new file mode 100644 index 0000000..f1cb2ab --- /dev/null +++ b/pkg/app/export.go @@ -0,0 +1,7 @@ +package app + +import "path/filepath" + +func (app *App) moduleName(srcDir string) string { + return "example.com/" + filepath.Base(srcDir) +} diff --git a/pkg/app/load.go b/pkg/app/load.go index e85f928..461826b 100644 --- a/pkg/app/load.go +++ b/pkg/app/load.go @@ -2,12 +2,13 @@ package app import ( "fmt" - "io/ioutil" "net/url" "os" "path/filepath" "strings" + fs2 "github.com/mumoshu/variant2/pkg/fs" + "github.com/hashicorp/hcl/v2/ext/typeexpr" "github.com/hashicorp/hcl/v2" @@ -26,11 +27,11 @@ type configurable struct { Body hcl.Body } -func loadFiles(filenames ...string) (map[string][]byte, error) { +func loadFiles(fs *fs2.FileSystem, filenames ...string) (map[string][]byte, error) { srcs := map[string][]byte{} for _, filename := range filenames { - src, err := ioutil.ReadFile(filename) + src, err := fs.ReadFile(filename) if err != nil { return nil, err } @@ -108,7 +109,9 @@ type Setup func() (*Instance, error) func FromFile(path string) Setup { return func() (*Instance, error) { - srcs, err := loadFiles(path) + fs := &fs2.FileSystem{} + + srcs, err := loadFiles(fs, path) if err != nil { return nil, err } @@ -124,12 +127,14 @@ func FromFile(path string) Setup { func FromDir(dir string) Setup { return func() (*Instance, error) { - fs, err := findVariantFiles(dir) + fs := &fs2.FileSystem{} + + files, err := findVariantFiles(fs, dir) if err != nil { return nil, err } - srcs, err := loadFiles(fs...) + srcs, err := loadFiles(fs, files...) if err != nil { return nil, err } @@ -188,7 +193,7 @@ func NewImportFunc(importBaseDir string, f func(string) (*App, error)) func(stri } } -func findVariantFiles(dirPathOrURL string) ([]string, error) { +func findVariantFiles(fs *fs2.FileSystem, dirPathOrURL string) ([]string, error) { var dir string s := strings.Split(dirPathOrURL, "::") @@ -230,7 +235,7 @@ func findVariantFiles(dirPathOrURL string) ([]string, error) { dir = dirPathOrURL } - files, err := conf.FindVariantFiles(dir) + files, err := conf.FindVariantFiles(fs, dir) if err != nil { return nil, fmt.Errorf("failed to get %s files: %v", conf.VariantFileExt, err) } diff --git a/pkg/conf/load.go b/pkg/conf/load.go index 465ff1d..2813ac5 100644 --- a/pkg/conf/load.go +++ b/pkg/conf/load.go @@ -1,8 +1,9 @@ package conf import ( - "os" "path/filepath" + + "github.com/mumoshu/variant2/pkg/fs" ) const ( @@ -11,19 +12,19 @@ const ( // FindVariantFiles walks the given path and returns the files ending whose ext is .variant // Also, it returns the path if the path is just a file and a HCL file -func FindVariantFiles(path string) ([]string, error) { +func FindVariantFiles(fs *fs.FileSystem, path string) ([]string, error) { var ( files []string err error ) - fi, err := os.Stat(path) + fi, err := fs.Stat(path) if err != nil { return files, err } if fi.IsDir() { - found, err := filepath.Glob(filepath.Join(path, "*"+VariantFileExt+"*")) + found, err := fs.Glob(filepath.Join(path, "*"+VariantFileExt+"*")) if err != nil { return nil, err } @@ -35,7 +36,7 @@ func FindVariantFiles(path string) ([]string, error) { continue } - info, err := os.Stat(f) + info, err := fs.Stat(f) if err != nil { return nil, err diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go new file mode 100644 index 0000000..f3d95ae --- /dev/null +++ b/pkg/fs/fs.go @@ -0,0 +1,165 @@ +package fs + +import ( + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "sort" + "sync" + + "github.com/rakyll/statik/fs" +) + +const ( + VendorPrefix = "vendored" +) + +type FileSystem struct { + sync.Once + fs http.FileSystem +} + +type noopFS struct { +} + +func (f *noopFS) Open(_ string) (http.File, error) { + return nil, os.ErrNotExist +} + +func (s *FileSystem) ReadFile(path string) ([]byte, error) { + fs, err := s.getFS() + if err != nil { + return nil, err + } + + f, err := fs.Open(s.vendored(path)) + if err == os.ErrNotExist { + return ioutil.ReadFile(path) + } + defer f.Close() + + bs, err := ioutil.ReadAll(f) + if err != nil { + return nil, fmt.Errorf("reading statik file: %w", err) + } + + return bs, nil +} + +func (s *FileSystem) Stat(path string) (os.FileInfo, error) { + fs, err := s.getFS() + if err != nil { + return nil, err + } + + f, err := fs.Open(s.vendored(path)) + if err == os.ErrNotExist { + return os.Stat(path) + } + defer f.Close() + + return f.Stat() +} + +func (s *FileSystem) Glob(pattern string) ([]string, error) { + fs, err := s.getFS() + if err != nil { + return nil, err + } + + dir, _ := filepath.Split(s.vendored(pattern)) + + found, err := glob(fs, dir, s.vendored(pattern)) + if err != nil { + return nil, fmt.Errorf("glob using statik: %w", err) + } + + if len(found) > 0 { + return found, nil + } + + found, err = filepath.Glob(pattern) + if err != nil { + return nil, fmt.Errorf("glob using filepath: %w", err) + } + + return found, nil +} + +func (s *FileSystem) getFS() (http.FileSystem, error) { + var err error + + s.Once.Do(func() { + s.fs, err = fs.New() + if err != nil { + s.fs = &noopFS{} + err = nil + } + }) + + if err != nil { + return nil, err + } + + return s.fs, nil +} + +func (s *FileSystem) vendored(path string) string { + return filepath.Join(string(filepath.Separator), path) +} + +func glob(fs http.FileSystem, dir, pattern string) ([]string, error) { + d, err := fs.Open(dir) + if err != nil { + return nil, nil + } + defer d.Close() + + fi, err := d.Stat() + if err != nil { + return nil, nil + } + + if !fi.IsDir() { + return nil, nil + } + + entries, err := d.Readdir(-1) + if err != nil { + return nil, fmt.Errorf("readdir: %w", err) + } + + var names []string + + for _, ent := range entries { + if ent.IsDir() { + subEntries, err := glob(fs, "/"+ent.Name(), pattern) + if err != nil { + return nil, err + } + + names = append(names, subEntries...) + } else { + names = append(names, filepath.Join(dir, ent.Name())) + } + } + + sort.Strings(names) + + var m []string + + for _, n := range names { + matched, err := filepath.Match(pattern, n) + if err != nil { + return m, err + } + + if matched { + m = append(m, n) + } + } + + return m, nil +}