Add additional output formats (tarball and layout) (ko-build#134)

* Create a MultiPublisher

MultiPublisher mimics io.MultiWriter in that it will publish an image to
multiple publish.Interface implementations.

* Add publish.{Tarball,Layout}Publisher

This adds support for publishing in the tarball format and to an OCI
image layout.

The tarball format isn't great, yet. It only supports writing once
instead of appending.

* Consolidate options

These were spread all over the place for no reasons. Now all the
publisher related options are grouped together.

* Add options for tarball/layout

Adds --oci-layout-path, --tarball, and --push flags.

--push=false will disable the default behavior of publishing to a
registry.

* go mod vendor

* Add Close method to publish.Interface

This allows us to defer writing to the tarball until we've collected all
the images that have been published.

* Fix tests

pkg/commands/apply.go

@@ -30,10 +30,8 @@ import (
 // addApply augments our CLI surface with apply.
 func addApply(topLevel *cobra.Command) {
 	koApplyFlags := []string{}
-	lo := &options.LocalOptions{}
-	no := &options.NameOptions{}
+	po := &options.PublishOptions{}
 	fo := &options.FilenameOptions{}
-	ta := &options.TagsOptions{}
 	so := &options.SelectorOptions{}
 	sto := &options.StrictOptions{}
 	bo := &options.BuildOptions{}
@@ -75,10 +73,11 @@ func addApply(topLevel *cobra.Command) {
 			if err != nil {
 				log.Fatalf("error creating builder: %v", err)
 			}
-			publisher, err := makePublisher(no, lo, ta)
+			publisher, err := makePublisher(po)
 			if err != nil {
 				log.Fatalf("error creating publisher: %v", err)
 			}
+			defer publisher.Close()
 			// Create a set of ko-specific flags to ignore when passing through
 			// kubectl global flags.
 			ignoreSet := make(map[string]struct{})
@@ -145,10 +144,8 @@ func addApply(topLevel *cobra.Command) {
 			}
 		},
 	}
-	options.AddLocalArg(apply, lo)
-	options.AddNamingArgs(apply, no)
+	options.AddPublishArg(apply, po)
 	options.AddFileArg(apply, fo)
-	options.AddTagsArg(apply, ta)
 	options.AddSelectorArg(apply, so)
 	options.AddStrictArg(apply, sto)
 	options.AddBuildOptions(apply, bo)

pkg/commands/create.go

@@ -30,10 +30,8 @@ import (
 // addCreate augments our CLI surface with apply.
 func addCreate(topLevel *cobra.Command) {
 	koCreateFlags := []string{}
-	lo := &options.LocalOptions{}
-	no := &options.NameOptions{}
+	po := &options.PublishOptions{}
 	fo := &options.FilenameOptions{}
-	ta := &options.TagsOptions{}
 	so := &options.SelectorOptions{}
 	sto := &options.StrictOptions{}
 	bo := &options.BuildOptions{}
@@ -75,10 +73,11 @@ func addCreate(topLevel *cobra.Command) {
 			if err != nil {
 				log.Fatalf("error creating builder: %v", err)
 			}
-			publisher, err := makePublisher(no, lo, ta)
+			publisher, err := makePublisher(po)
 			if err != nil {
 				log.Fatalf("error creating publisher: %v", err)
 			}
+			defer publisher.Close()
 			// Create a set of ko-specific flags to ignore when passing through
 			// kubectl global flags.
 			ignoreSet := make(map[string]struct{})
@@ -145,10 +144,8 @@ func addCreate(topLevel *cobra.Command) {
 			}
 		},
 	}
-	options.AddLocalArg(create, lo)
-	options.AddNamingArgs(create, no)
+	options.AddPublishArg(create, po)
 	options.AddFileArg(create, fo)
-	options.AddTagsArg(create, ta)
 	options.AddSelectorArg(create, so)
 	options.AddStrictArg(create, sto)
 	options.AddBuildOptions(create, bo)

pkg/commands/options/flatname.go → pkg/commands/options/publish.go

@@ -23,18 +23,43 @@ import (
 	"github.com/spf13/cobra"
 )
 
-// NameOptions represents options for the ko binary.
-type NameOptions struct {
+// PublishOptions encapsulates options when publishing.
+type PublishOptions struct {
+	Tags []string
+
+	// Push publishes images to a registry.
+	Push bool
+
+	// Local publishes images to a local docker daemon.
+	Local            bool
+	InsecureRegistry bool
+
+	OCILayoutPath string
+	TarballFile   string
+
 	// PreserveImportPaths preserves the full import path after KO_DOCKER_REPO.
 	PreserveImportPaths bool
 	// BaseImportPaths uses the base path without MD5 hash after KO_DOCKER_REPO.
 	BaseImportPaths bool
 }
 
-func AddNamingArgs(cmd *cobra.Command, no *NameOptions) {
-	cmd.Flags().BoolVarP(&no.PreserveImportPaths, "preserve-import-paths", "P", no.PreserveImportPaths,
+func AddPublishArg(cmd *cobra.Command, po *PublishOptions) {
+	cmd.Flags().StringSliceVarP(&po.Tags, "tags", "t", []string{"latest"},
+		"Which tags to use for the produced image instead of the default 'latest' tag.")
+
+	cmd.Flags().BoolVar(&po.Push, "push", true, "Push images to KO_DOCKER_REPO")
+
+	cmd.Flags().BoolVarP(&po.Local, "local", "L", po.Local,
+		"Load into images to local docker daemon.")
+	cmd.Flags().BoolVar(&po.InsecureRegistry, "insecure-registry", po.InsecureRegistry,
+		"Whether to skip TLS verification on the registry")
+
+	cmd.Flags().StringVar(&po.OCILayoutPath, "oci-layout-path", "", "Path to save the OCI image layout of the built images")
+	cmd.Flags().StringVar(&po.TarballFile, "tarball", "", "File to save images tarballs")
+
+	cmd.Flags().BoolVarP(&po.PreserveImportPaths, "preserve-import-paths", "P", po.PreserveImportPaths,
 		"Whether to preserve the full import path after KO_DOCKER_REPO.")
-	cmd.Flags().BoolVarP(&no.BaseImportPaths, "base-import-paths", "B", no.BaseImportPaths,
+	cmd.Flags().BoolVarP(&po.BaseImportPaths, "base-import-paths", "B", po.BaseImportPaths,
 		"Whether to use the base path without MD5 hash after KO_DOCKER_REPO.")
 }
 
@@ -52,10 +77,10 @@ func baseImportPaths(importpath string) string {
 	return filepath.Base(importpath)
 }
 
-func MakeNamer(no *NameOptions) publish.Namer {
-	if no.PreserveImportPaths {
+func MakeNamer(po *PublishOptions) publish.Namer {
+	if po.PreserveImportPaths {
 		return preserveImportPath
-	} else if no.BaseImportPaths {
+	} else if po.BaseImportPaths {
 		return baseImportPaths
 	}
 	return packageWithMD5

pkg/commands/publish.go

@@ -24,9 +24,7 @@ import (
 
 // addPublish augments our CLI surface with publish.
 func addPublish(topLevel *cobra.Command) {
-	lo := &options.LocalOptions{}
-	no := &options.NameOptions{}
-	ta := &options.TagsOptions{}
+	po := &options.PublishOptions{}
 	bo := &options.BuildOptions{}
 
 	publish := &cobra.Command{
@@ -64,10 +62,11 @@ func addPublish(topLevel *cobra.Command) {
 			if err != nil {
 				log.Fatalf("error creating builder: %v", err)
 			}
-			publisher, err := makePublisher(no, lo, ta)
+			publisher, err := makePublisher(po)
 			if err != nil {
 				log.Fatalf("error creating publisher: %v", err)
 			}
+			defer publisher.Close()
 			ctx := createCancellableContext()
 			images, err := publishImages(ctx, args, publisher, builder)
 			if err != nil {
@@ -78,9 +77,7 @@ func addPublish(topLevel *cobra.Command) {
 			}
 		},
 	}
-	options.AddLocalArg(publish, lo)
-	options.AddNamingArgs(publish, no)
-	options.AddTagsArg(publish, ta)
+	options.AddPublishArg(publish, po)
 	options.AddBuildOptions(publish, bo)
 	topLevel.AddCommand(publish)
 }

pkg/commands/resolve.go

@@ -24,10 +24,8 @@ import (
 
 // addResolve augments our CLI surface with resolve.
 func addResolve(topLevel *cobra.Command) {
-	lo := &options.LocalOptions{}
-	no := &options.NameOptions{}
+	po := &options.PublishOptions{}
 	fo := &options.FilenameOptions{}
-	ta := &options.TagsOptions{}
 	so := &options.SelectorOptions{}
 	sto := &options.StrictOptions{}
 	bo := &options.BuildOptions{}
@@ -62,20 +60,19 @@ func addResolve(topLevel *cobra.Command) {
 			if err != nil {
 				log.Fatalf("error creating builder: %v", err)
 			}
-			publisher, err := makePublisher(no, lo, ta)
+			publisher, err := makePublisher(po)
 			if err != nil {
 				log.Fatalf("error creating publisher: %v", err)
 			}
+			defer publisher.Close()
 			ctx := createCancellableContext()
 			if err := resolveFilesToWriter(ctx, builder, publisher, fo, so, sto, os.Stdout); err != nil {
 				log.Fatal(err)
 			}
 		},
 	}
-	options.AddLocalArg(resolve, lo)
-	options.AddNamingArgs(resolve, no)
+	options.AddPublishArg(resolve, po)
 	options.AddFileArg(resolve, fo)
-	options.AddTagsArg(resolve, ta)
 	options.AddSelectorArg(resolve, so)
 	options.AddStrictArg(resolve, sto)
 	options.AddBuildOptions(resolve, bo)

pkg/commands/resolver.go

@@ -111,16 +111,19 @@ func makeBuilder(bo *options.BuildOptions) (*build.Caching, error) {
 	return build.NewCaching(innerBuilder)
 }
 
-func makePublisher(no *options.NameOptions, lo *options.LocalOptions, ta *options.TagsOptions) (publish.Interface, error) {
+func makePublisher(po *options.PublishOptions) (publish.Interface, error) {
 	// Create the publish.Interface that we will use to publish image references
 	// to either a docker daemon or a container image registry.
 	innerPublisher, err := func() (publish.Interface, error) {
-		namer := options.MakeNamer(no)
-
 		repoName := os.Getenv("KO_DOCKER_REPO")
-		if lo.Local || repoName == publish.LocalDomain {
-			return publish.NewDaemon(namer, ta.Tags), nil
+		namer := options.MakeNamer(po)
+		if repoName == publish.LocalDomain || po.Local {
+			// TODO(jonjohnsonjr): I'm assuming that nobody will
+			// use local with other publishers, but that might
+			// not be true.
+			return publish.NewDaemon(namer, po.Tags), nil
 		}
+
 		if repoName == "" {
 			return nil, errors.New("KO_DOCKER_REPO environment variable is unset")
 		}
@@ -130,12 +133,31 @@ func makePublisher(no *options.NameOptions, lo *options.LocalOptions, ta *option
 			}
 		}
 
-		return publish.NewDefault(repoName,
-			publish.WithTransport(defaultTransport()),
-			publish.WithAuthFromKeychain(authn.DefaultKeychain),
-			publish.WithNamer(namer),
-			publish.WithTags(ta.Tags),
-			publish.Insecure(lo.InsecureRegistry))
+		publishers := []publish.Interface{}
+		if po.OCILayoutPath != "" {
+			lp, err := publish.NewLayout(po.OCILayoutPath)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create LayoutPublisher for %q: %v", po.OCILayoutPath, err)
+			}
+			publishers = append(publishers, lp)
+		}
+		if po.TarballFile != "" {
+			tp := publish.NewTarball(po.TarballFile, repoName, namer, po.Tags)
+			publishers = append(publishers, tp)
+		}
+		if po.Push {
+			dp, err := publish.NewDefault(repoName,
+				publish.WithTransport(defaultTransport()),
+				publish.WithAuthFromKeychain(authn.DefaultKeychain),
+				publish.WithNamer(namer),
+				publish.WithTags(po.Tags),
+				publish.Insecure(po.InsecureRegistry))
+			if err != nil {
+				return nil, err
+			}
+			publishers = append(publishers, dp)
+		}
+		return publish.MultiPublisher(publishers...), nil
 	}()
 	if err != nil {
 		return nil, err

pkg/commands/run.go

@@ -27,9 +27,7 @@ import (
 
 // addRun augments our CLI surface with run.
 func addRun(topLevel *cobra.Command) {
-	lo := &options.LocalOptions{}
-	no := &options.NameOptions{}
-	ta := &options.TagsOptions{}
+	po := &options.PublishOptions{}
 	bo := &options.BuildOptions{}
 
 	run := &cobra.Command{
@@ -69,10 +67,11 @@ func addRun(topLevel *cobra.Command) {
 			if err != nil {
 				log.Fatalf("error creating builder: %v", err)
 			}
-			publisher, err := makePublisher(no, lo, ta)
+			publisher, err := makePublisher(po)
 			if err != nil {
 				log.Fatalf("error creating publisher: %v", err)
 			}
+			defer publisher.Close()
 
 			if len(os.Args) < 3 {
 				log.Fatalf("usage: %s run <package>", os.Args[0])
@@ -137,9 +136,7 @@ func addRun(topLevel *cobra.Command) {
 			UnknownFlags: true,
 		},
 	}
-	options.AddLocalArg(run, lo)
-	options.AddNamingArgs(run, no)
-	options.AddTagsArg(run, ta)
+	options.AddPublishArg(run, po)
 	options.AddBuildOptions(run, bo)
 
 	topLevel.AddCommand(run)

pkg/internal/testing/fixed.go

@@ -72,6 +72,10 @@ func (f *fixedPublish) Publish(_ v1.Image, s string) (name.Reference, error) {
 	return &d, nil
 }
 
+func (f *fixedPublish) Close() error {
+	return nil
+}
+
 func ComputeDigest(base name.Repository, ref string, h v1.Hash) string {
 	d, err := name.NewDigest(fmt.Sprintf("%s/%s@%s", base, ref, h))
 	if err != nil {

pkg/publish/daemon.go

@@ -78,3 +78,7 @@ func (d *demon) Publish(img v1.Image, s string) (name.Reference, error) {
 
 	return &digestTag, nil
 }
+
+func (d *demon) Close() error {
+	return nil
+}

pkg/publish/default.go

@@ -140,3 +140,7 @@ func (d *defalt) Publish(img v1.Image, s string) (name.Reference, error) {
 	log.Printf("Published %v", dig)
 	return &dig, nil
 }
+
+func (d *defalt) Close() error {
+	return nil
+}

pkg/publish/layout.go

@@ -0,0 +1,66 @@
+// Copyright 2020 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package publish
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/google/go-containerregistry/pkg/name"
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/empty"
+	"github.com/google/go-containerregistry/pkg/v1/layout"
+)
+
+type LayoutPublisher struct {
+	p layout.Path
+}
+
+// NewLayout returns a new publish.Interface that saves images to an OCI Image Layout.
+func NewLayout(path string) (Interface, error) {
+	p, err := layout.FromPath(path)
+	if err != nil {
+		p, err = layout.Write(path, empty.Index)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return &LayoutPublisher{p}, nil
+}
+
+// Publish implements publish.Interface.
+func (l *LayoutPublisher) Publish(img v1.Image, s string) (name.Reference, error) {
+	log.Printf("Saving %v", s)
+	if err := l.p.AppendImage(img); err != nil {
+		return nil, err
+	}
+	log.Printf("Saved %v", s)
+
+	h, err := img.Digest()
+	if err != nil {
+		return nil, err
+	}
+
+	dig, err := name.NewDigest(fmt.Sprintf("%s@%s", l.p, h))
+	if err != nil {
+		return nil, err
+	}
+
+	return dig, nil
+}
+
+func (l *LayoutPublisher) Close() error {
+	return nil
+}