Multi-platform ko (ko-build#38)

🦜

go.mod

@@ -6,23 +6,32 @@ require (
 	github.com/docker/cli v0.0.0-20200303162255-7d407207c304 // indirect
 	github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7
 	github.com/dprotaso/go-yit v0.0.0-20191028211022-135eb7262960
-	github.com/fsnotify/fsnotify v1.4.7
+	github.com/fsnotify/fsnotify v1.4.9
+	github.com/go-logr/logr v0.2.1 // indirect
 	github.com/go-training/helloworld v0.0.0-20200225145412-ba5f4379d78b
-	github.com/google/go-cmp v0.3.0
-	github.com/google/go-containerregistry v0.0.0-20200310013544-4fe717a9b4cb
+	github.com/google/go-cmp v0.4.1
+	github.com/google/go-containerregistry v0.1.3
+	github.com/googleapis/gnostic v0.4.0 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect
+	github.com/json-iterator/go v1.1.10 // indirect
 	github.com/mattmoor/dep-notify v0.0.0-20190205035814-a45dec370a17
+	github.com/maxbrunsfeld/counterfeiter/v6 v6.2.3 // indirect
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.4.0
-	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
-	golang.org/x/tools v0.0.0-20200210192313-1ace956b0e17
-	gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71
+	golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208
+	golang.org/x/text v0.3.3 // indirect
+	golang.org/x/tools v0.0.0-20200924205911-8a9a89368bd3
+	gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776
 	gotest.tools/v3 v3.0.2 // indirect
 	k8s.io/apimachinery v0.18.8
 	k8s.io/cli-runtime v0.18.8
+	k8s.io/code-generator v0.19.0-alpha.1 // indirect
+	k8s.io/gengo v0.0.0-20200728071708-7794989d0000 // indirect
+	k8s.io/klog/v2 v2.3.0 // indirect
 	sigs.k8s.io/kind v0.8.1
+	sigs.k8s.io/structured-merge-diff/v4 v4.0.1 // indirect
 )
 
 replace (

pkg/build/build.go

@@ -18,6 +18,7 @@ import (
 	"context"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/types"
 )
 
 // Interface abstracts different methods for turning a supported importpath
@@ -28,6 +29,19 @@ type Interface interface {
 	// TODO(mattmoor): Verify that some base repo: foo.io/bar can be suffixed with this reference and parsed.
 	IsSupportedReference(string) bool
 
-	// Build turns the given importpath reference into a v1.Image containing the Go binary.
-	Build(context.Context, string) (v1.Image, error)
+	// Build turns the given importpath reference into a v1.Image containing the Go binary
+	// (or a set of images as a v1.ImageIndex).
+	Build(context.Context, string) (Result, error)
 }
+
+// Result represents the product of a Build. This is usually a v1.Image or v1.ImageIndex.
+type Result interface {
+	MediaType() (types.MediaType, error)
+	Size() (int64, error)
+	Digest() (v1.Hash, error)
+	RawManifest() ([]byte, error)
+}
+
+// Assert that Image and ImageIndex implement Result.
+var _ Result = (v1.Image)(nil)
+var _ Result = (v1.ImageIndex)(nil)

pkg/build/future.go

@@ -16,11 +16,9 @@ package build
 
 import (
 	"sync"
-
-	v1 "github.com/google/go-containerregistry/pkg/v1"
 )
 
-func newFuture(work func() (v1.Image, error)) *future {
+func newFuture(work func() (Result, error)) *future {
 	// Create a channel on which to send the result.
 	ch := make(chan *result)
 	// Initiate the actual work, sending its result
@@ -40,7 +38,7 @@ func newFuture(work func() (v1.Image, error)) *future {
 }
 
 type result struct {
-	img v1.Image
+	img Result
 	err error
 }
 
@@ -52,7 +50,7 @@ type future struct {
 }
 
 // Get blocks on the result of the future.
-func (f *future) Get() (v1.Image, error) {
+func (f *future) Get() (Result, error) {
 	// Block on the promise of a result until we get one.
 	result, ok := <-f.promise
 	if ok {

pkg/build/future_test.go

@@ -17,15 +17,14 @@ package build
 import (
 	"testing"
 
-	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/random"
 )
 
-func makeImage() (v1.Image, error) {
-	return random.Image(256, 8)
+func makeImage() (Result, error) {
+	return random.Index(256, 8, 1)
 }
 
-func digest(t *testing.T, img v1.Image) string {
+func digest(t *testing.T, img Result) string {
 	d, err := img.Digest()
 	if err != nil {
 		t.Fatalf("Digest() = %v", err)

pkg/build/gobuild.go

@@ -33,17 +33,20 @@ import (
 	"strings"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
 	"github.com/google/go-containerregistry/pkg/v1/mutate"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
+	"github.com/google/go-containerregistry/pkg/v1/types"
 )
 
 const (
 	appDir             = "/ko-app"
 	defaultAppFilename = "ko-app"
 )
 
-// GetBase takes an importpath and returns a base v1.Image.
-type GetBase func(string) (v1.Image, error)
+// GetBase takes an importpath and returns a base image.
+type GetBase func(string) (Result, error)
+
 type builder func(context.Context, string, v1.Platform, bool) (string, error)
 
 type buildContext interface {
@@ -254,7 +257,7 @@ func build(ctx context.Context, ip string, platform v1.Platform, disableOptimiza
 	cmd.Stderr = &output
 	cmd.Stdout = &output
 
-	log.Printf("Building %s", ip)
+	log.Printf("Building %s for %s", ip, platform.Architecture)
 	if err := cmd.Run(); err != nil {
 		os.RemoveAll(tmpDir)
 		log.Printf("Unexpected error running \"go build\": %v\n%v", err, output.String())
@@ -442,15 +445,9 @@ func (g *gobuild) tarKoData(ref reference) (*bytes.Buffer, error) {
 	return buf, walkRecursive(tw, root, kodataRoot)
 }
 
-// Build implements build.Interface
-func (gb *gobuild) Build(ctx context.Context, s string) (v1.Image, error) {
+func (gb *gobuild) buildOne(ctx context.Context, s string, base v1.Image) (v1.Image, error) {
 	ref := newRef(s)
 
-	// Determine the appropriate base image for this import path.
-	base, err := gb.getBase(ref.Path())
-	if err != nil {
-		return nil, err
-	}
 	cf, err := base.ConfigFile()
 	if err != nil {
 		return nil, err
@@ -563,3 +560,77 @@ func updatePath(cf *v1.ConfigFile) {
 	// If we get here, we never saw PATH.
 	cf.Config.Env = append(cf.Config.Env, "PATH="+appDir)
 }
+
+// Build implements build.Interface
+func (gb *gobuild) Build(ctx context.Context, s string) (Result, error) {
+	// Determine the appropriate base image for this import path.
+	base, err := gb.getBase(s)
+	if err != nil {
+		return nil, err
+	}
+
+	// Determine what kind of base we have and if we should publish an image or an index.
+	mt, err := base.MediaType()
+	if err != nil {
+		return nil, err
+	}
+
+	switch mt {
+	case types.OCIImageIndex, types.DockerManifestList:
+		base, ok := base.(v1.ImageIndex)
+		if !ok {
+			return nil, fmt.Errorf("failed to interpret base as index: %v", base)
+		}
+		return gb.buildAll(ctx, s, base)
+	case types.OCIManifestSchema1, types.DockerManifestSchema2:
+		base, ok := base.(v1.Image)
+		if !ok {
+			return nil, fmt.Errorf("failed to interpret base as image: %v", base)
+		}
+		return gb.buildOne(ctx, s, base)
+	default:
+		return nil, fmt.Errorf("base image media type: %s", mt)
+	}
+}
+
+// TODO(#192): Do these in parallel?
+func (gb *gobuild) buildAll(ctx context.Context, s string, base v1.ImageIndex) (v1.ImageIndex, error) {
+	im, err := base.IndexManifest()
+	if err != nil {
+		return nil, err
+	}
+
+	// Build an image for each child from the base and append it to a new index to produce the result.
+	adds := []mutate.IndexAddendum{}
+	for _, desc := range im.Manifests {
+		// Nested index is pretty rare. We could support this in theory, but return an error for now.
+		if desc.MediaType != types.OCIManifestSchema1 && desc.MediaType != types.DockerManifestSchema2 {
+			return nil, fmt.Errorf("%q has unexpected mediaType %q in base for %q", desc.Digest, desc.MediaType, s)
+		}
+
+		base, err := base.Image(desc.Digest)
+		if err != nil {
+			return nil, err
+		}
+		img, err := gb.buildOne(ctx, s, base)
+		if err != nil {
+			return nil, err
+		}
+		adds = append(adds, mutate.IndexAddendum{
+			Add: img,
+			Descriptor: v1.Descriptor{
+				URLs:        desc.URLs,
+				MediaType:   desc.MediaType,
+				Annotations: desc.Annotations,
+				Platform:    desc.Platform,
+			},
+		})
+	}
+
+	baseType, err := base.MediaType()
+	if err != nil {
+		return nil, err
+	}
+
+	return mutate.IndexMediaType(mutate.AppendManifests(empty.Index, adds...), baseType), nil
+}