Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[modules] Dependent command packages can now be built with ko #154

Merged
merged 2 commits into from
May 1, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 65 additions & 14 deletions pkg/build/gobuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,17 @@ const (
type GetBase func(string) (v1.Image, error)
type builder func(context.Context, string, v1.Platform, bool) (string, error)

type buildContext interface {
Import(path string, srcDir string, mode gb.ImportMode) (*gb.Package, error)
}

type gobuild struct {
getBase GetBase
creationTime v1.Time
build builder
disableOptimizations bool
mod *modInfo
mod *modules
buildContext buildContext
}

// Option is a functional option for NewGo.
Expand All @@ -62,7 +67,8 @@ type gobuildOpener struct {
creationTime v1.Time
build builder
disableOptimizations bool
mod *modInfo
mod *modules
buildContext buildContext
}

func (gbo *gobuildOpener) Open() (Interface, error) {
Expand All @@ -75,38 +81,80 @@ func (gbo *gobuildOpener) Open() (Interface, error) {
build: gbo.build,
disableOptimizations: gbo.disableOptimizations,
mod: gbo.mod,
buildContext: gbo.buildContext,
}, nil
}

// https://golang.org/pkg/cmd/go/internal/modinfo/#ModulePublic
type modules struct {
main *modInfo
deps map[string]*modInfo
}

type modInfo struct {
Path string
Dir string
Main bool
}

// moduleInfo returns the module path and module root directory for a project
// using go modules, otherwise returns nil.
//
// Related: https://github.com/golang/go/issues/26504
func moduleInfo() *modInfo {
output, err := exec.Command("go", "list", "-mod=readonly", "-m", "-json").Output()
func moduleInfo() (*modules, error) {
modules := modules{
deps: make(map[string]*modInfo),
}

// TODO we read all the output as a single byte array - it may
// be possible & more efficient to stream it
output, err := exec.Command("go", "list", "-mod=readonly", "-json", "-m", "all").Output()
if err != nil {
return nil
return nil, nil
}
var info modInfo
if err := json.Unmarshal(output, &info); err != nil {
return nil

dec := json.NewDecoder(bytes.NewReader(output))

for {
var info modInfo

err := dec.Decode(&info)
if err == io.EOF {
// all done
break
}

modules.deps[info.Path] = &info

if info.Main {
modules.main = &info
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

couldn't we hit N of these? Which one wins?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I believe there's only one

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

main != package main but specifically "main module" = root module

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that's not confusing at all 🙄

}

if err != nil {
return nil, fmt.Errorf("error reading module data %w", err)
}
}

if modules.main == nil {
return nil, fmt.Errorf("couldn't find main module")
}
return &info

return &modules, nil
}

// NewGo returns a build.Interface implementation that:
// 1. builds go binaries named by importpath,
// 2. containerizes the binary on a suitable base,
func NewGo(options ...Option) (Interface, error) {
module, err := moduleInfo()
if err != nil {
return nil, err
}

gbo := &gobuildOpener{
build: build,
mod: moduleInfo(),
build: build,
mod: module,
buildContext: &gb.Default,
}

for _, option := range options {
Expand Down Expand Up @@ -141,16 +189,19 @@ func (g *gobuild) IsSupportedReference(s string) bool {
// Note that we will fall back to GOPATH if the project isn't using go modules.
func (g *gobuild) importPackage(ref reference) (*gb.Package, error) {
if g.mod == nil {
return gb.Import(ref.Path(), gb.Default.GOPATH, gb.ImportComment)
return g.buildContext.Import(ref.Path(), gb.Default.GOPATH, gb.ImportComment)
}

// If we're inside a go modules project, try to use the module's directory
// as our source root to import:
// * any strict reference we get
// * paths that match module path prefix (they should be in this project)
// * relative paths (they should also be in this project)
if ref.IsStrict() || strings.HasPrefix(ref.Path(), g.mod.Path) || gb.IsLocalImport(ref.Path()) {
return gb.Import(ref.Path(), g.mod.Dir, gb.ImportComment)
// * path is a module

_, isDep := g.mod.deps[ref.Path()]
if ref.IsStrict() || strings.HasPrefix(ref.Path(), g.mod.main.Path) || gb.IsLocalImport(ref.Path()) || isDep {
return g.buildContext.Import(ref.Path(), g.mod.main.Dir, gb.ImportComment)
}

return nil, fmt.Errorf("unmatched importPackage %q with gomodules", ref.String())
Expand Down
35 changes: 31 additions & 4 deletions pkg/build/gobuild_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package build
import (
"archive/tar"
"context"
gb "go/build"
"io"
"io/ioutil"
"path"
Expand Down Expand Up @@ -69,19 +70,39 @@ func TestGoBuildIsSupportedRefWithModules(t *testing.T) {
if err != nil {
t.Fatalf("random.Image() = %v", err)
}
mod := &modInfo{
Path: "github.com/google/ko/cmd/ko/test",
Dir: ".",

mods := &modules{
main: &modInfo{
Path: "github.com/google/ko/cmd/ko/test",
Dir: ".",
},
deps: map[string]*modInfo{
"github.com/some/module/cmd": &modInfo{
Path: "github.com/some/module/cmd",
Dir: ".",
},
},
}

opts := []Option{
WithBaseImages(func(string) (v1.Image, error) { return base, nil }),
withModuleInfo(mods),
withBuildContext(stubBuildContext{
// make all referenced deps commands
"github.com/google/ko/cmd/ko/test": &gb.Package{Name: "main"},
"github.com/some/module/cmd": &gb.Package{Name: "main"},
}),
}

ng, err := NewGo(WithBaseImages(func(string) (v1.Image, error) { return base, nil }), withModuleInfo(mod))
ng, err := NewGo(opts...)
if err != nil {
t.Fatalf("NewGo() = %v", err)
}

// Supported import paths.
for _, importpath := range []string{
"github.com/google/ko/cmd/ko/test", // ko can build the test package.
"github.com/some/module/cmd", // ko can build commands in dependent modules
} {
t.Run(importpath, func(t *testing.T) {
if !ng.IsSupportedReference(importpath) {
Expand Down Expand Up @@ -376,3 +397,9 @@ func TestGoBuild(t *testing.T) {
}
})
}

type stubBuildContext map[string]*gb.Package

func (s stubBuildContext) Import(path string, srcDir string, mode gb.ImportMode) (*gb.Package, error) {
return s[path], nil
}
11 changes: 9 additions & 2 deletions pkg/build/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,16 @@ func withBuilder(b builder) Option {
// withModulePath is a functional option for overriding the module path for
// the current ko invocation.
// This is exposed for testing.
func withModuleInfo(mi *modInfo) Option {
func withModuleInfo(m *modules) Option {
return func(gbo *gobuildOpener) error {
gbo.mod = mi
gbo.mod = m
return nil
}
}

func withBuildContext(b buildContext) Option {
return func(gbo *gobuildOpener) error {
gbo.buildContext = b
return nil
}
}