From 103ff5b2a89121a20398b133246a8e8277ecfe00 Mon Sep 17 00:00:00 2001 From: Halvard Skogsrud Date: Thu, 21 Oct 2021 16:14:06 +1100 Subject: [PATCH] Use build config Dir for all go tool commands Ensure that the directory specified in build configs in `.ko.yaml` is used to: 1. Load module information 2. Resolve local paths to Go import paths 3. Working directory for compilation The change achieves this by introducing `gobuilds`, which contains a map of import path to `build.Interface` instances. Each entry maps to a `builds` entry from `.ko.yaml`. `gobuilds` dispatches to the builder instances based on the requested import path, and falls back to a default builder if there's no match. Thanks to @jonjohnsonjr for the suggestions in https://github.com/google/ko/issues/422#issuecomment-909408527 Also removes mutable globals in the `commands` package. Fixes: #422 --- README.md | 21 +- integration_test.sh | 46 ++++- pkg/build/config.go | 2 +- pkg/build/gobuilds.go | 155 ++++++++++++++ pkg/build/gobuilds_test.go | 190 ++++++++++++++++++ pkg/commands/config.go | 116 +---------- pkg/commands/config_test.go | 68 +------ pkg/commands/options/build.go | 122 ++++++++++- pkg/commands/options/build_test.go | 89 ++++++++ .../{ => options}/testdata/config/.ko.yaml | 0 .../{ => options}/testdata/paths/.ko.yaml | 0 .../testdata/paths/app/cmd/foo/main.go | 0 .../{ => options}/testdata/paths/app/go.mod | 0 pkg/commands/resolver.go | 16 +- 14 files changed, 626 insertions(+), 199 deletions(-) create mode 100644 pkg/build/gobuilds.go create mode 100644 pkg/build/gobuilds_test.go create mode 100644 pkg/commands/options/build_test.go rename pkg/commands/{ => options}/testdata/config/.ko.yaml (100%) rename pkg/commands/{ => options}/testdata/paths/.ko.yaml (100%) rename pkg/commands/{ => options}/testdata/paths/app/cmd/foo/main.go (100%) rename pkg/commands/{ => options}/testdata/paths/app/go.mod (100%) diff --git a/README.md b/README.md index 2a6b87d42e..ccd4a00b41 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,7 @@ configuration section in your `.ko.yaml`. ```yaml builds: - id: foo + dir: . main: ./foobar/foo env: - GOPRIVATE=git.internal.example.com,source.developers.google.com @@ -146,7 +147,8 @@ builds: - -extldflags "-static" - -X main.version={{.Env.VERSION}} - id: bar - main: ./foobar/bar/main.go + dir: ./bar + main: . env: - GOCACHE=/workspace/.gocache ldflags: @@ -154,15 +156,22 @@ builds: - -w ``` -For the build, `ko` will pick the entry based on the respective import path -being used. It will be matched against the local path that is configured using -`dir` and `main`. In the context of `ko`, it is fine just to specify `main` -with the intended import path. +If your repository contains multiple modules (multiple `go.mod` files in +different directories), use the `dir` field to specify the directory where +`ko` should run `go build`. + +`ko` picks the entry from `builds` based on the import path you request. The +import path is matched against the result of joining `dir` and `main`. + +The paths specified in `dir` and `main` are relative to the working directory +of the `ko` process. + +The `ldflags` default value is `[]`. _Please note:_ Even though the configuration section is similar to the [GoReleaser `builds` section](https://goreleaser.com/customization/build/), only the `env`, `flags` and `ldflags` fields are currently supported. Also, the -templating support is currently limited to environment variables only. +templating support is currently limited to using environment variables only. ## Naming Images diff --git a/integration_test.sh b/integration_test.sh index 7e0c7853e3..c51747f185 100755 --- a/integration_test.sh +++ b/integration_test.sh @@ -3,7 +3,7 @@ set -o errexit set -o nounset set -o pipefail -ROOT_DIR=$(dirname $0) +ROOT_DIR=$(dirname "$0") pushd "$ROOT_DIR" @@ -11,7 +11,8 @@ ROOT_DIR="$(pwd)" echo "Moving GOPATH into /tmp/ to test modules behavior." export ORIGINAL_GOPATH="$GOPATH" -export GOPATH="$(mktemp -d)" +GOPATH="$(mktemp -d)" +export GOPATH pushd "$GOPATH" || exit 1 @@ -91,8 +92,49 @@ fi echo "7. On outside the module should fail." pushd .. || exit 1 GO111MODULE=on ./ko/ko build --local github.com/go-training/helloworld && exit 1 +popd || exit 1 +echo "8. On outside with build config specifying the test module builds." +ko_exec_dir=$(pwd) +pushd "$(mktemp -d)" || exit 1 +for app in foo bar ; do + mkdir -p $app/cmd || exit 1 + pushd $app || exit 1 + GO111MODULE=on go mod init example.com/$app || exit 1 + cat << EOF > ./cmd/main.go || exit 1 +package main + +import "fmt" + +func main() { + fmt.Println("$app") +} +EOF + popd || exit 1 +done +cat << EOF > .ko.yaml || exit 1 +builds: +- id: foo-app + dir: ./foo + main: ./cmd +- id: bar-app + dir: ./bar + main: ./cmd +EOF +for app in foo bar ; do + # test both local and fully qualified import paths + for prefix in example.com . ; do + import_path=$prefix/$app/cmd + RESULT="$(GO111MODULE=on GOFLAGS="" "$ko_exec_dir"/ko publish --local $import_path | grep "$FILTER" | xargs -I% docker run %)" + if [[ "$RESULT" != *"$app"* ]]; then + echo "Test FAILED for $import_path. Saw $RESULT but expected $app" && exit 1 + else + echo "Test PASSED for $import_path" + fi + done +done popd || exit 1 + popd || exit 1 popd || exit 1 diff --git a/pkg/build/config.go b/pkg/build/config.go index 5e1f4cf7b3..ae5a9a0cbc 100644 --- a/pkg/build/config.go +++ b/pkg/build/config.go @@ -62,7 +62,7 @@ func (a *FlagArray) UnmarshalYAML(unmarshal func(interface{}) error) error { // the original GoReleaser name to match better with the ko naming. // // TODO: Introduce support for more fields where possible and where it makes -/// sense for `ko`, for example ModTimestamp, Env, or GoBinary. +/// sense for `ko`, for example ModTimestamp or GoBinary. // type Config struct { // ID only serves as an identifier internally diff --git a/pkg/build/gobuilds.go b/pkg/build/gobuilds.go new file mode 100644 index 0000000000..283533cd34 --- /dev/null +++ b/pkg/build/gobuilds.go @@ -0,0 +1,155 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "context" + "fmt" + gb "go/build" + "path" + "path/filepath" + "strings" +) + +type gobuilds struct { + // Map of fully qualified import path to go builder with config + builders map[string]builderWithConfig + + // Default go builder used if there's no matching build config. + defaultBuilder Interface + + // workingDirectory is typically ".", but it may be a different value if ko is embedded as a library. + workingDirectory string +} + +// builderWithConfig is not an imaginative name. +type builderWithConfig struct { + builder Interface + config Config +} + +// NewGobuilds returns a build.Interface that can dispatch to builders based on matching the import path to a build config in .ko.yaml. +func NewGobuilds(ctx context.Context, workingDirectory string, buildConfigs map[string]Config, opts ...Option) (Interface, error) { + if workingDirectory == "" { + workingDirectory = "." + } + defaultBuilder, err := NewGo(ctx, workingDirectory, opts...) + if err != nil { + return nil, fmt.Errorf("could not create default go builder: %w", err) + } + g := &gobuilds{ + builders: map[string]builderWithConfig{}, + defaultBuilder: defaultBuilder, + workingDirectory: workingDirectory, + } + for importpath, buildConfig := range buildConfigs { + builderDirectory := path.Join(workingDirectory, buildConfig.Dir) + builder, err := NewGo(ctx, builderDirectory, opts...) + if err != nil { + return nil, fmt.Errorf("could not create go builder for config (%q): %w", importpath, err) + } + g.builders[importpath] = builderWithConfig{ + builder: builder, + config: buildConfig, + } + } + return g, nil +} + +// QualifyImport implements build.Interface +func (g *gobuilds) QualifyImport(importpath string) (string, error) { + b := g.builder(importpath) + if b.config.Dir != "" { + var err error + importpath, err = relativePath(b.config.Dir, importpath) + if err != nil { + return "", err + } + } + return b.builder.QualifyImport(importpath) +} + +// IsSupportedReference implements build.Interface +func (g *gobuilds) IsSupportedReference(importpath string) error { + return g.builder(importpath).builder.IsSupportedReference(importpath) +} + +// Build implements build.Interface +func (g *gobuilds) Build(ctx context.Context, importpath string) (Result, error) { + return g.builder(importpath).builder.Build(ctx, importpath) +} + +// builder selects a go builder for the provided import path. +// The `importpath` argument can be either local (e.g., `./cmd/foo`) or not (e.g., `example.com/app/cmd/foo`). +func (g *gobuilds) builder(importpath string) builderWithConfig { + importpath = strings.TrimPrefix(importpath, StrictScheme) + if len(g.builders) == 0 { + return builderWithConfig{ + builder: g.defaultBuilder, + } + } + // first, try to find go builder by fully qualified import path + if builderWithConfig, exists := g.builders[importpath]; exists { + return builderWithConfig + } + // second, try to find go builder by local path + for _, builderWithConfig := range g.builders { + // Match go builder by trying to resolve the local path to a fully qualified import path. If successful, we have a winner. + relPath, err := relativePath(builderWithConfig.config.Dir, importpath) + if err != nil { + // Cannot determine a relative path. Move on and try the next go builder. + continue + } + _, err = builderWithConfig.builder.QualifyImport(relPath) + if err != nil { + // There's an error turning the local path into a fully qualified import path. Move on and try the next go builder. + continue + } + return builderWithConfig + } + // fall back to default go builder + return builderWithConfig{ + builder: g.defaultBuilder, + } +} + +// relativePath takes as input a local import path, and returns a path relative to the base directory. +// +// For example, given the following inputs: +// - baseDir: "app" +// - importpath: "./app/cmd/foo +// The output is: "./cmd/foo" +// +// If the input is a not a local import path as determined by go/build.IsLocalImport(), the input is returned unchanged. +// +// If the import path is _not_ a subdirectory of baseDir, the result is an error. +func relativePath(baseDir string, importpath string) (string, error) { + // Return input unchanged if the import path is a fully qualified import path + if !gb.IsLocalImport(importpath) { + return importpath, nil + } + relPath, err := filepath.Rel(baseDir, importpath) + if err != nil { + return "", fmt.Errorf("cannot determine relative path of baseDir (%q) and local path (%q): %v", baseDir, importpath, err) + } + if strings.HasPrefix(relPath, "..") { + // TODO Is this assumption correct? + return "", fmt.Errorf("import path (%q) must be a subdirectory of build config directory (%q)", importpath, baseDir) + } + if !strings.HasPrefix(relPath, ".") && relPath != "." { + relPath = "./" + relPath // ensure go/build.IsLocalImport() interprets this as a local path + } + return relPath, nil +} diff --git a/pkg/build/gobuilds_test.go b/pkg/build/gobuilds_test.go new file mode 100644 index 0000000000..c601974a94 --- /dev/null +++ b/pkg/build/gobuilds_test.go @@ -0,0 +1,190 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "context" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/random" +) + +func Test_gobuilds(t *testing.T) { + base, err := random.Image(1024, 1) + if err != nil { + t.Fatalf("random.Image() = %v", err) + } + baseRef := name.MustParseReference("all.your/base") + opts := []Option{ + WithBaseImages(func(context.Context, string) (name.Reference, Result, error) { return baseRef, base, nil }), + withBuilder(writeTempFile), + } + + tests := []struct { + description string + workingDirectory string + buildConfigs map[string]Config + opts []Option + nilDefaultBuilder bool // set to true if you want to test build config and don't want the test to fall back to the default builder + importpath string + }{ + { + description: "default builder used when no build configs provided", + opts: opts, + importpath: "github.com/google/ko", + }, + { + description: "match build config using fully qualified import path", + workingDirectory: "../..", + buildConfigs: map[string]Config{ + "github.com/google/ko/test": { + ID: "build-config-0", + Dir: "test", + }, + }, + nilDefaultBuilder: true, + opts: opts, + importpath: "github.com/google/ko/test", + }, + { + description: "match build config using ko scheme-prefixed fully qualified import path", + workingDirectory: "../..", + buildConfigs: map[string]Config{ + "github.com/google/ko/test": { + ID: "build-config-1", + Dir: "test", + }, + }, + nilDefaultBuilder: true, + opts: opts, + importpath: "ko://github.com/google/ko/test", + }, + { + description: "find build config by resolving local import path to fully qualified import path", + workingDirectory: "../../test", + buildConfigs: map[string]Config{ + "github.com/google/ko/test": { + ID: "build-config-2", + }, + }, + nilDefaultBuilder: true, + opts: opts, + importpath: ".", + }, + { + description: "find build config by matching local import path to build config directory", + workingDirectory: "../..", + buildConfigs: map[string]Config{ + "github.com/google/ko/tes12t": { + ID: "build-config-3", + Dir: "test", + }, + }, + nilDefaultBuilder: true, + opts: opts, + importpath: "./test", + }, + } + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + ctx := context.Background() + bi, err := NewGobuilds(ctx, test.workingDirectory, test.buildConfigs, test.opts...) + if err != nil { + t.Fatalf("NewGobuilds(): unexpected error: %v", err) + } + gbs := bi.(*gobuilds) + if test.nilDefaultBuilder { + gbs.defaultBuilder = nil + } + qualifiedImportpath, err := gbs.QualifyImport(test.importpath) + if err != nil { + t.Fatalf("gobuilds.QualifyImport(%s): unexpected error: %v", test.importpath, err) + } + if err = gbs.IsSupportedReference(qualifiedImportpath); err != nil { + t.Fatalf("gobuilds.IsSupportedReference(%s): unexpected error: %v", qualifiedImportpath, err) + } + result, err := gbs.Build(ctx, qualifiedImportpath) + if err != nil { + t.Fatalf("gobuilds.Build(%s): unexpected error = %v", qualifiedImportpath, err) + } + if result == nil { + t.Fatalf("gobuilds.Build(%s): expected non-nil result", qualifiedImportpath) + } + }) + } +} + +func Test_relativePath(t *testing.T) { + tests := []struct { + description string + baseDir string + importpath string + want string + wantErr bool + }{ + { + description: "all empty string", + }, + { + description: "all current directory", + baseDir: ".", + importpath: ".", + want: ".", + }, + { + description: "fully qualified import path without ko prefix", + baseDir: "also-any-value-because-it-is-ignored", + importpath: "example.com/app/cmd/foo", + want: "example.com/app/cmd/foo", + }, + { + description: "fully qualified import path with ko prefix", + baseDir: "also-any-value-because-it-is-ignored", + importpath: "ko://example.com/app/cmd/foo", + want: "ko://example.com/app/cmd/foo", + }, + { + description: "importpath is local subdirectory", + baseDir: "foo", + importpath: "./foo/bar", + want: "./bar", + }, + { + description: "importpath is same local directory", + baseDir: "foo/bar", + importpath: "./foo/bar", + want: ".", + }, + { + description: "importpath is not subdirectory or same local directory", + baseDir: "foo", + importpath: "./bar", + wantErr: true, + }, + } + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + got, err := relativePath(test.baseDir, test.importpath) + if (err != nil) != test.wantErr { + t.Errorf("relativePath() error = %v, wantErr %v", err, test.wantErr) + return + } + if got != test.want { + t.Errorf("relativePath() got = %v, want %v", got, test.want) + } + }) + } +} diff --git a/pkg/commands/config.go b/pkg/commands/config.go index e576972388..71b061a4e4 100644 --- a/pkg/commands/config.go +++ b/pkg/commands/config.go @@ -22,7 +22,6 @@ import ( "log" "os" "os/signal" - "path/filepath" "strconv" "strings" "syscall" @@ -34,26 +33,13 @@ import ( "github.com/google/go-containerregistry/pkg/v1/daemon" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/types" + "github.com/google/ko/pkg/build" "github.com/google/ko/pkg/commands/options" "github.com/google/ko/pkg/publish" - "github.com/spf13/viper" - "golang.org/x/tools/go/packages" -) - -const ( - // configDefaultBaseImage is the default base image if not specified in .ko.yaml. - configDefaultBaseImage = "gcr.io/distroless/static:nonroot" -) - -var ( - defaultBaseImage string - baseImageOverrides map[string]string - buildConfigs map[string]build.Config ) // getBaseImage returns a function that determines the base image for a given import path. -// If the `bo.BaseImage` parameter is non-empty, it overrides base image configuration from `.ko.yaml`. func getBaseImage(platform string, bo *options.BuildOptions) build.GetBase { return func(ctx context.Context, s string) (name.Reference, build.Result, error) { s = strings.TrimPrefix(s, build.StrictScheme) @@ -63,14 +49,11 @@ func getBaseImage(platform string, bo *options.BuildOptions) build.GetBase { // github.com/GoogleCloudPlatform/foo/cmd/bar // comes through as: // github.com/googlecloudplatform/foo/cmd/bar - baseImage, ok := baseImageOverrides[strings.ToLower(s)] - if !ok { - baseImage = defaultBaseImage - } - if bo.BaseImage != "" { + baseImage, ok := bo.BaseImageOverrides[strings.ToLower(s)] + if !ok || baseImage == "" { baseImage = bo.BaseImage } - nameOpts := []name.Option{} + var nameOpts []name.Option if bo.InsecureRegistry { nameOpts = append(nameOpts, name.Insecure) } @@ -172,94 +155,3 @@ func createCancellableContext() context.Context { return ctx } - -func createBuildConfigMap(workingDirectory string, configs []build.Config) (map[string]build.Config, error) { - buildConfigsByImportPath := make(map[string]build.Config) - for i, config := range configs { - // Make sure to behave like GoReleaser by defaulting to the current - // directory in case the build or main field is not set, check - // https://goreleaser.com/customization/build/ for details - if config.Dir == "" { - config.Dir = "." - } - if config.Main == "" { - config.Main = "." - } - - // baseDir is the directory where `go list` will be run to look for package information - baseDir := filepath.Join(workingDirectory, config.Dir) - - // To behave like GoReleaser, check whether the configured `main` config value points to a - // source file, and if so, just use the directory it is in - path := config.Main - if fi, err := os.Stat(filepath.Join(baseDir, config.Main)); err == nil && fi.Mode().IsRegular() { - path = filepath.Dir(config.Main) - } - - // By default, paths configured in the builds section are considered - // local import paths, therefore add a "./" equivalent as a prefix to - // the constructured import path - localImportPath := fmt.Sprint(".", string(filepath.Separator), path) - - pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName, Dir: baseDir}, localImportPath) - if err != nil { - return nil, fmt.Errorf("'builds': entry #%d does not contain a valid local import path (%s) for directory (%s): %v", i, localImportPath, baseDir, err) - } - - if len(pkgs) != 1 { - return nil, fmt.Errorf("'builds': entry #%d results in %d local packages, only 1 is expected", i, len(pkgs)) - } - importPath := pkgs[0].PkgPath - buildConfigsByImportPath[importPath] = config - } - - return buildConfigsByImportPath, nil -} - -// loadConfig reads build configuration from defaults, environment variables, and the `.ko.yaml` config file. -func loadConfig(workingDirectory string) error { - v := viper.New() - if workingDirectory == "" { - workingDirectory = "." - } - // If omitted, use this base image. - v.SetDefault("defaultBaseImage", configDefaultBaseImage) - v.SetConfigName(".ko") // .yaml is implicit - v.SetEnvPrefix("KO") - v.AutomaticEnv() - - if override := os.Getenv("KO_CONFIG_PATH"); override != "" { - v.AddConfigPath(override) - } - - v.AddConfigPath(workingDirectory) - - if err := v.ReadInConfig(); err != nil { - if _, ok := err.(viper.ConfigFileNotFoundError); !ok { - return fmt.Errorf("error reading config file: %v", err) - } - } - - ref := v.GetString("defaultBaseImage") - if _, err := name.ParseReference(ref); err != nil { - return fmt.Errorf("'defaultBaseImage': error parsing %q as image reference: %v", ref, err) - } - defaultBaseImage = ref - - baseImageOverrides = make(map[string]string) - overrides := v.GetStringMapString("baseImageOverrides") - for key, value := range overrides { - if _, err := name.ParseReference(value); err != nil { - return fmt.Errorf("'baseImageOverrides': error parsing %q as image reference: %v", value, err) - } - baseImageOverrides[key] = value - } - - var builds []build.Config - if err := v.UnmarshalKey("builds", &builds); err != nil { - return fmt.Errorf("configuration section 'builds' cannot be parsed") - } - var err error - buildConfigs, err = createBuildConfigMap(workingDirectory, builds) - return err -} diff --git a/pkg/commands/config_test.go b/pkg/commands/config_test.go index c86d529180..b62abfb221 100644 --- a/pkg/commands/config_test.go +++ b/pkg/commands/config_test.go @@ -22,7 +22,7 @@ import ( "testing" "github.com/google/go-containerregistry/pkg/crane" - "github.com/google/ko/pkg/build" + "github.com/google/ko/pkg/commands/options" ) @@ -58,69 +58,3 @@ func TestOverrideDefaultBaseImageUsingBuildOption(t *testing.T) { t.Errorf("got digest %s, wanted %s", gotDigest, wantDigest) } } - -// TestDefaultBaseImage is a canary-type test for ensuring that config has been read when creating a builder. -func TestDefaultBaseImage(t *testing.T) { - _, err := NewBuilder(context.Background(), &options.BuildOptions{ - WorkingDirectory: "testdata/config", - }) - if err != nil { - t.Fatal(err) - } - wantDefaultBaseImage := "gcr.io/distroless/base:nonroot" // matches value in ./testdata/.ko.yaml - if defaultBaseImage != wantDefaultBaseImage { - t.Fatalf("wanted defaultBaseImage %s, got %s", wantDefaultBaseImage, defaultBaseImage) - } -} - -func TestBuildConfigWithWorkingDirectoryAndDirAndMain(t *testing.T) { - _, err := NewBuilder(context.Background(), &options.BuildOptions{ - WorkingDirectory: "testdata/paths", - }) - if err != nil { - t.Fatalf("NewBuilder(): %+v", err) - } - - if len(buildConfigs) != 1 { - t.Fatalf("expected 1 build config, got %d", len(buildConfigs)) - } - expectedImportPath := "example.com/testapp/cmd/foo" // module from app/go.mod + `main` from .ko.yaml - if _, exists := buildConfigs[expectedImportPath]; !exists { - t.Fatalf("expected build config for import path [%s], got %+v", expectedImportPath, buildConfigs) - } -} - -func TestCreateBuildConfigs(t *testing.T) { - compare := func(expected string, actual string) { - if expected != actual { - t.Errorf("test case failed: expected '%#v', but actual value is '%#v'", expected, actual) - } - } - - buildConfigs := []build.Config{ - {ID: "defaults"}, - {ID: "OnlyMain", Main: "test"}, - {ID: "OnlyMainWithFile", Main: "test/main.go"}, - {ID: "OnlyDir", Dir: "test"}, - {ID: "DirAndMain", Dir: "test", Main: "main.go"}, - } - - for _, b := range buildConfigs { - buildConfigMap, err := createBuildConfigMap("../..", []build.Config{b}) - if err != nil { - t.Fatal(err) - } - for importPath, buildCfg := range buildConfigMap { - switch buildCfg.ID { - case "defaults": - compare("github.com/google/ko", importPath) - - case "OnlyMain", "OnlyMainWithFile", "OnlyDir", "DirAndMain": - compare("github.com/google/ko/test", importPath) - - default: - t.Fatalf("unknown test case: %s", buildCfg.ID) - } - } - } -} diff --git a/pkg/commands/options/build.go b/pkg/commands/options/build.go index 5533655ff9..5910487626 100644 --- a/pkg/commands/options/build.go +++ b/pkg/commands/options/build.go @@ -17,8 +17,21 @@ limitations under the License. package options import ( - "github.com/google/ko/pkg/build" + "fmt" + "os" + "path/filepath" + + "github.com/google/go-containerregistry/pkg/name" "github.com/spf13/cobra" + "github.com/spf13/viper" + "golang.org/x/tools/go/packages" + + "github.com/google/ko/pkg/build" +) + +const ( + // configDefaultBaseImage is the default base image if not specified in .ko.yaml. + configDefaultBaseImage = "gcr.io/distroless/static:nonroot" ) // BuildOptions represents options for the ko builder. @@ -27,6 +40,9 @@ type BuildOptions struct { // If non-empty, this takes precedence over the value in `.ko.yaml`. BaseImage string + // BaseImageOverrides stores base image overrides for import paths. + BaseImageOverrides map[string]string + // WorkingDirectory allows for setting the working directory for invocations of the `go` tool. // Empty string means the current working directory. WorkingDirectory string @@ -41,7 +57,7 @@ type BuildOptions struct { InsecureRegistry bool - // BuildConfigs enables programmatic overriding of build config set in `.ko.yaml`. + // BuildConfigs stores the per-image build config from `.ko.yaml`. BuildConfigs map[string]build.Config } @@ -55,3 +71,105 @@ func AddBuildOptions(cmd *cobra.Command, bo *BuildOptions) { cmd.Flags().StringSliceVar(&bo.Labels, "image-label", []string{}, "Which labels (key=value) to add to the image.") } + +// LoadConfig reads build configuration from defaults, environment variables, and the `.ko.yaml` config file. +func (bo *BuildOptions) LoadConfig() error { + v := viper.New() + if bo.WorkingDirectory == "" { + bo.WorkingDirectory = "." + } + // If omitted, use this base image. + v.SetDefault("defaultBaseImage", configDefaultBaseImage) + v.SetConfigName(".ko") // .yaml is implicit + v.SetEnvPrefix("KO") + v.AutomaticEnv() + + if override := os.Getenv("KO_CONFIG_PATH"); override != "" { + v.AddConfigPath(override) + } + + v.AddConfigPath(bo.WorkingDirectory) + + if err := v.ReadInConfig(); err != nil { + if _, ok := err.(viper.ConfigFileNotFoundError); !ok { + return fmt.Errorf("error reading config file: %v", err) + } + } + + if bo.BaseImage == "" { + ref := v.GetString("defaultBaseImage") + if _, err := name.ParseReference(ref); err != nil { + return fmt.Errorf("'defaultBaseImage': error parsing %q as image reference: %v", ref, err) + } + bo.BaseImage = ref + } + + if len(bo.BaseImageOverrides) == 0 { + baseImageOverrides := map[string]string{} + overrides := v.GetStringMapString("baseImageOverrides") + for key, value := range overrides { + if _, err := name.ParseReference(value); err != nil { + return fmt.Errorf("'baseImageOverrides': error parsing %q as image reference: %v", value, err) + } + baseImageOverrides[key] = value + } + bo.BaseImageOverrides = baseImageOverrides + } + + if len(bo.BuildConfigs) == 0 { + var builds []build.Config + if err := v.UnmarshalKey("builds", &builds); err != nil { + return fmt.Errorf("configuration section 'builds' cannot be parsed") + } + buildConfigs, err := createBuildConfigMap(bo.WorkingDirectory, builds) + if err != nil { + return fmt.Errorf("could not create build config map: %w", err) + } + bo.BuildConfigs = buildConfigs + } + + return nil +} + +func createBuildConfigMap(workingDirectory string, configs []build.Config) (map[string]build.Config, error) { + buildConfigsByImportPath := make(map[string]build.Config) + for i, config := range configs { + // Make sure to behave like GoReleaser by defaulting to the current + // directory in case the build or main field is not set, check + // https://goreleaser.com/customization/build/ for details + if config.Dir == "" { + config.Dir = "." + } + if config.Main == "" { + config.Main = "." + } + + // baseDir is the directory where `go list` will be run to look for package information + baseDir := filepath.Join(workingDirectory, config.Dir) + + // To behave like GoReleaser, check whether the configured `main` config value points to a + // source file, and if so, just use the directory it is in + path := config.Main + if fi, err := os.Stat(filepath.Join(baseDir, config.Main)); err == nil && fi.Mode().IsRegular() { + path = filepath.Dir(config.Main) + } + + // By default, paths configured in the builds section are considered + // local import paths, therefore add a "./" equivalent as a prefix to + // the constructured import path + localImportPath := fmt.Sprint(".", string(filepath.Separator), path) + + pkgs, err := packages.Load(&packages.Config{Mode: packages.NeedName, Dir: baseDir}, localImportPath) + if err != nil { + return nil, fmt.Errorf("'builds': entry #%d does not contain a valid local import path (%s) for directory (%s): %v", i, localImportPath, baseDir, err) + } + + if len(pkgs) != 1 { + return nil, fmt.Errorf("'builds': entry #%d results in %d local packages, only 1 is expected", i, len(pkgs)) + } + importPath := pkgs[0].PkgPath + buildConfigsByImportPath[importPath] = config + } + + return buildConfigsByImportPath, nil +} diff --git a/pkg/commands/options/build_test.go b/pkg/commands/options/build_test.go new file mode 100644 index 0000000000..a8acfb9674 --- /dev/null +++ b/pkg/commands/options/build_test.go @@ -0,0 +1,89 @@ +// Copyright 2021 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package options + +import ( + "testing" + + "github.com/google/ko/pkg/build" +) + +func TestDefaultBaseImage(t *testing.T) { + bo := &BuildOptions{ + WorkingDirectory: "testdata/config", + } + err := bo.LoadConfig() + if err != nil { + t.Fatal(err) + } + + wantDefaultBaseImage := "gcr.io/distroless/base:nonroot" // matches value in ./testdata/.ko.yaml + if bo.BaseImage != wantDefaultBaseImage { + t.Fatalf("wanted BaseImage %s, got %s", wantDefaultBaseImage, bo.BaseImage) + } +} + +func TestBuildConfigWithWorkingDirectoryAndDirAndMain(t *testing.T) { + bo := &BuildOptions{ + WorkingDirectory: "testdata/paths", + } + err := bo.LoadConfig() + if err != nil { + t.Fatalf("NewBuilder(): %v", err) + } + + if len(bo.BuildConfigs) != 1 { + t.Fatalf("expected 1 build config, got %d", len(bo.BuildConfigs)) + } + expectedImportPath := "example.com/testapp/cmd/foo" // module from app/go.mod + `main` from .ko.yaml + if _, exists := bo.BuildConfigs[expectedImportPath]; !exists { + t.Fatalf("expected build config for import path [%s], got %+v", expectedImportPath, bo.BuildConfigs) + } +} + +func TestCreateBuildConfigs(t *testing.T) { + compare := func(expected string, actual string) { + if expected != actual { + t.Errorf("test case failed: expected '%#v', but actual value is '%#v'", expected, actual) + } + } + + buildConfigs := []build.Config{ + {ID: "defaults"}, + {ID: "OnlyMain", Main: "test"}, + {ID: "OnlyMainWithFile", Main: "test/main.go"}, + {ID: "OnlyDir", Dir: "test"}, + {ID: "DirAndMain", Dir: "test", Main: "main.go"}, + } + + for _, b := range buildConfigs { + buildConfigMap, err := createBuildConfigMap("../../..", []build.Config{b}) + if err != nil { + t.Fatal(err) + } + for importPath, buildCfg := range buildConfigMap { + switch buildCfg.ID { + case "defaults": + compare("github.com/google/ko", importPath) + + case "OnlyMain", "OnlyMainWithFile", "OnlyDir", "DirAndMain": + compare("github.com/google/ko/test", importPath) + + default: + t.Fatalf("unknown test case: %s", buildCfg.ID) + } + } + } +} diff --git a/pkg/commands/testdata/config/.ko.yaml b/pkg/commands/options/testdata/config/.ko.yaml similarity index 100% rename from pkg/commands/testdata/config/.ko.yaml rename to pkg/commands/options/testdata/config/.ko.yaml diff --git a/pkg/commands/testdata/paths/.ko.yaml b/pkg/commands/options/testdata/paths/.ko.yaml similarity index 100% rename from pkg/commands/testdata/paths/.ko.yaml rename to pkg/commands/options/testdata/paths/.ko.yaml diff --git a/pkg/commands/testdata/paths/app/cmd/foo/main.go b/pkg/commands/options/testdata/paths/app/cmd/foo/main.go similarity index 100% rename from pkg/commands/testdata/paths/app/cmd/foo/main.go rename to pkg/commands/options/testdata/paths/app/cmd/foo/main.go diff --git a/pkg/commands/testdata/paths/app/go.mod b/pkg/commands/options/testdata/paths/app/go.mod similarity index 100% rename from pkg/commands/testdata/paths/app/go.mod rename to pkg/commands/options/testdata/paths/app/go.mod diff --git a/pkg/commands/resolver.go b/pkg/commands/resolver.go index bac8cd53bc..84a907be85 100644 --- a/pkg/commands/resolver.go +++ b/pkg/commands/resolver.go @@ -32,14 +32,15 @@ import ( "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" - "github.com/google/ko/pkg/build" - "github.com/google/ko/pkg/commands/options" - "github.com/google/ko/pkg/publish" - "github.com/google/ko/pkg/resolve" "github.com/mattmoor/dep-notify/pkg/graph" "golang.org/x/sync/errgroup" "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/labels" + + "github.com/google/ko/pkg/build" + "github.com/google/ko/pkg/commands/options" + "github.com/google/ko/pkg/publish" + "github.com/google/ko/pkg/resolve" ) // ua returns the ko user agent. @@ -106,11 +107,8 @@ func gobuildOptions(bo *options.BuildOptions) ([]build.Option, error) { opts = append(opts, build.WithLabel(parts[0], parts[1])) } - // prefer buildConfigs from BuildOptions if bo.BuildConfigs != nil { opts = append(opts, build.WithConfig(bo.BuildConfigs)) - } else if len(buildConfigs) > 0 { - opts = append(opts, build.WithConfig(buildConfigs)) } return opts, nil @@ -122,14 +120,14 @@ func NewBuilder(ctx context.Context, bo *options.BuildOptions) (build.Interface, } func makeBuilder(ctx context.Context, bo *options.BuildOptions) (*build.Caching, error) { - if err := loadConfig(bo.WorkingDirectory); err != nil { + if err := bo.LoadConfig(); err != nil { return nil, err } opt, err := gobuildOptions(bo) if err != nil { return nil, fmt.Errorf("error setting up builder options: %v", err) } - innerBuilder, err := build.NewGo(ctx, bo.WorkingDirectory, opt...) + innerBuilder, err := build.NewGobuilds(ctx, bo.WorkingDirectory, bo.BuildConfigs, opt...) if err != nil { return nil, err }