From c9d34ce4957346118eeb94779f632c5a938f2271 Mon Sep 17 00:00:00 2001 From: Jon Johnson Date: Wed, 12 Feb 2020 10:59:32 -0800 Subject: [PATCH 1/7] Create a MultiPublisher MultiPublisher mimics io.MultiWriter in that it will publish an image to multiple publish.Interface implementations. --- pkg/publish/multi.go | 45 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 pkg/publish/multi.go diff --git a/pkg/publish/multi.go b/pkg/publish/multi.go new file mode 100644 index 0000000000..4ffa8d7744 --- /dev/null +++ b/pkg/publish/multi.go @@ -0,0 +1,45 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package publish + +import ( + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// MultiPublisher creates a publisher that publishes to all +// the provided publishers, similar to the Unix tee(1) command. +// +// When calling Publish, the name.Reference returned will be the return value +// of the last publisher passed to MultiPublisher (last one wins). +func MultiPublisher(publishers ...Interface) Interface { + return &multiPublisher{publishers} +} + +type multiPublisher struct { + publishers []Interface +} + +// Publish implements publish.Interface. +func (p *multiPublisher) Publish(img v1.Image, s string) (ref name.Reference, err error) { + for _, pub := range p.publishers { + ref, err = pub.Publish(img, s) + if err != nil { + return + } + } + + return +} From 606c0d08f930bb9f71ca7dddc45599c2d8763e41 Mon Sep 17 00:00:00 2001 From: Jon Johnson Date: Wed, 12 Feb 2020 15:01:08 -0800 Subject: [PATCH 2/7] Add publish.{Tarball,Layout}Publisher This adds support for publishing in the tarball format and to an OCI image layout. The tarball format isn't great, yet. It only supports writing once instead of appending. --- pkg/publish/layout.go | 62 +++++++++++++++++++++++++++ pkg/publish/layout_test.go | 48 +++++++++++++++++++++ pkg/publish/multi_test.go | 59 ++++++++++++++++++++++++++ pkg/publish/tarball.go | 84 +++++++++++++++++++++++++++++++++++++ pkg/publish/tarball_test.go | 69 ++++++++++++++++++++++++++++++ 5 files changed, 322 insertions(+) create mode 100644 pkg/publish/layout.go create mode 100644 pkg/publish/layout_test.go create mode 100644 pkg/publish/multi_test.go create mode 100644 pkg/publish/tarball.go create mode 100644 pkg/publish/tarball_test.go diff --git a/pkg/publish/layout.go b/pkg/publish/layout.go new file mode 100644 index 0000000000..a216f27a4d --- /dev/null +++ b/pkg/publish/layout.go @@ -0,0 +1,62 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package publish + +import ( + "fmt" + "log" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" +) + +type LayoutPublisher struct { + p layout.Path +} + +// NewLayout returns a new publish.Interface that saves images to an OCI Image Layout. +func NewLayout(path string) (Interface, error) { + p, err := layout.FromPath(path) + if err != nil { + p, err = layout.Write(path, empty.Index) + if err != nil { + return nil, err + } + } + return &LayoutPublisher{p}, nil +} + +// Publish implements publish.Interface. +func (l *LayoutPublisher) Publish(img v1.Image, s string) (name.Reference, error) { + log.Printf("Saving %v", s) + if err := l.p.AppendImage(img); err != nil { + return nil, err + } + log.Printf("Saved %v", s) + + h, err := img.Digest() + if err != nil { + return nil, err + } + + dig, err := name.NewDigest(fmt.Sprintf("%s@%s", l.p, h)) + if err != nil { + return nil, err + } + + return dig, nil +} diff --git a/pkg/publish/layout_test.go b/pkg/publish/layout_test.go new file mode 100644 index 0000000000..c15c2a9955 --- /dev/null +++ b/pkg/publish/layout_test.go @@ -0,0 +1,48 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package publish + +import ( + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/v1/random" +) + +func TestLayout(t *testing.T) { + img, err := random.Image(1024, 1) + if err != nil { + t.Fatalf("random.Image() = %v", err) + } + importpath := "github.com/Google/go-containerregistry/cmd/crane" + + tmp, err := ioutil.TempDir("", "ko") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + lp, err := NewLayout(tmp) + if err != nil { + t.Errorf("NewLayout() = %v", err) + } + if d, err := lp.Publish(img, importpath); err != nil { + t.Errorf("Publish() = %v", err) + } else if !strings.HasPrefix(d.String(), tmp) { + t.Errorf("Publish() = %v, wanted prefix %v", d, tmp) + } +} diff --git a/pkg/publish/multi_test.go b/pkg/publish/multi_test.go new file mode 100644 index 0000000000..99d4207920 --- /dev/null +++ b/pkg/publish/multi_test.go @@ -0,0 +1,59 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package publish + +import ( + "fmt" + "io/ioutil" + "os" + "testing" + + "github.com/google/go-containerregistry/pkg/v1/random" +) + +func TestMulti(t *testing.T) { + img, err := random.Image(1024, 1) + if err != nil { + t.Fatalf("random.Image() = %v", err) + } + base := "blah" + repoName := fmt.Sprintf("%s/%s", "example.com", base) + importpath := "github.com/Google/go-containerregistry/cmd/crane" + + fp, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + defer fp.Close() + defer os.Remove(fp.Name()) + + tp := NewTarball(fp.Name(), repoName, md5Hash, []string{}) + + tmp, err := ioutil.TempDir("", "ko") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + lp, err := NewLayout(tmp) + if err != nil { + t.Errorf("NewLayout() = %v", err) + } + + p := MultiPublisher(lp, tp) + if _, err := p.Publish(img, importpath); err != nil { + t.Errorf("Publish() = %v", err) + } +} diff --git a/pkg/publish/tarball.go b/pkg/publish/tarball.go new file mode 100644 index 0000000000..37895eb26e --- /dev/null +++ b/pkg/publish/tarball.go @@ -0,0 +1,84 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package publish + +import ( + "fmt" + "log" + "strings" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +type TarballPublisher struct { + file string + base string + namer Namer + tags []string +} + +// NewTarball returns a new publish.Interface that saves images to a tarball. +func NewTarball(file, base string, namer Namer, tags []string) *TarballPublisher { + return &TarballPublisher{file, base, namer, tags} +} + +// Publish implements publish.Interface. +func (t *TarballPublisher) Publish(img v1.Image, s string) (name.Reference, error) { + // https://github.com/google/go-containerregistry/issues/212 + s = strings.ToLower(s) + + refs := make(map[name.Reference]v1.Image) + for _, tagName := range t.tags { + tag, err := name.ParseReference(fmt.Sprintf("%s/%s:%s", t.base, t.namer(s), tagName)) + if err != nil { + return nil, err + } + refs[tag] = img + } + + h, err := img.Digest() + if err != nil { + return nil, err + } + + if len(t.tags) == 0 { + ref, err := name.ParseReference(fmt.Sprintf("%s/%s@%s", t.base, t.namer(s), h)) + if err != nil { + return nil, err + } + refs[ref] = img + } + + ref := fmt.Sprintf("%s/%s@%s", t.base, t.namer(s), h) + if len(t.tags) == 1 && t.tags[0] != defaultTags[0] { + // If a single tag is explicitly set (not latest), then this + // is probably a release, so include the tag in the reference. + ref = fmt.Sprintf("%s/%s:%s@%s", t.base, t.namer(s), t.tags[0], h) + } + dig, err := name.NewDigest(ref) + if err != nil { + return nil, err + } + + log.Printf("Saving %v", dig) + if err := tarball.MultiRefWriteToFile(t.file, refs); err != nil { + return nil, err + } + log.Printf("Saved %v", dig) + + return &dig, nil +} diff --git a/pkg/publish/tarball_test.go b/pkg/publish/tarball_test.go new file mode 100644 index 0000000000..07d7e897c5 --- /dev/null +++ b/pkg/publish/tarball_test.go @@ -0,0 +1,69 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package publish + +import ( + "fmt" + "io/ioutil" + "os" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/random" +) + +func TestTarball(t *testing.T) { + img, err := random.Image(1024, 1) + if err != nil { + t.Fatalf("random.Image() = %v", err) + } + base := "blah" + importpath := "github.com/Google/go-containerregistry/cmd/crane" + + fp, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err) + } + defer fp.Close() + defer os.Remove(fp.Name()) + + expectedRepo := fmt.Sprintf("%s/%s", base, md5Hash(strings.ToLower(importpath))) + + tag, err := name.NewTag(fmt.Sprintf("%s/%s:latest", "example.com", expectedRepo)) + if err != nil { + t.Fatalf("NewTag() = %v", err) + } + + repoName := fmt.Sprintf("%s/%s", "example.com", base) + tagss := [][]string{{ + // no tags + }, { + // one tag + "v0.1.0", + }, { + // multiple tags + "latest", + "debug", + }} + for _, tags := range tagss { + tp := NewTarball(fp.Name(), repoName, md5Hash, tags) + if d, err := tp.Publish(img, importpath); err != nil { + t.Errorf("Publish() = %v", err) + } else if !strings.HasPrefix(d.String(), tag.Repository.String()) { + t.Errorf("Publish() = %v, wanted prefix %v", d, tag.Repository) + } + } +} From f7871726603065256b3ca770351da653b3d8043f Mon Sep 17 00:00:00 2001 From: Jon Johnson Date: Wed, 12 Feb 2020 15:36:27 -0800 Subject: [PATCH 3/7] Consolidate options These were spread all over the place for no reasons. Now all the publisher related options are grouped together. --- pkg/commands/apply.go | 10 ++---- pkg/commands/create.go | 10 ++---- pkg/commands/options/binary.go | 30 ----------------- pkg/commands/options/local.go | 33 ------------------- .../options/{flatname.go => publish.go} | 30 ++++++++++++----- pkg/commands/options/tags.go | 29 ---------------- pkg/commands/publish.go | 10 ++---- pkg/commands/resolve.go | 10 ++---- pkg/commands/resolver.go | 12 +++---- pkg/commands/run.go | 10 ++---- 10 files changed, 43 insertions(+), 141 deletions(-) delete mode 100644 pkg/commands/options/binary.go delete mode 100644 pkg/commands/options/local.go rename pkg/commands/options/{flatname.go => publish.go} (60%) delete mode 100644 pkg/commands/options/tags.go diff --git a/pkg/commands/apply.go b/pkg/commands/apply.go index 204ef553b5..323823958f 100644 --- a/pkg/commands/apply.go +++ b/pkg/commands/apply.go @@ -30,10 +30,8 @@ import ( // addApply augments our CLI surface with apply. func addApply(topLevel *cobra.Command) { koApplyFlags := []string{} - lo := &options.LocalOptions{} - no := &options.NameOptions{} + po := &options.PublishOptions{} fo := &options.FilenameOptions{} - ta := &options.TagsOptions{} so := &options.SelectorOptions{} sto := &options.StrictOptions{} bo := &options.BuildOptions{} @@ -75,7 +73,7 @@ func addApply(topLevel *cobra.Command) { if err != nil { log.Fatalf("error creating builder: %v", err) } - publisher, err := makePublisher(no, lo, ta) + publisher, err := makePublisher(po) if err != nil { log.Fatalf("error creating publisher: %v", err) } @@ -145,10 +143,8 @@ func addApply(topLevel *cobra.Command) { } }, } - options.AddLocalArg(apply, lo) - options.AddNamingArgs(apply, no) + options.AddPublishArg(apply, po) options.AddFileArg(apply, fo) - options.AddTagsArg(apply, ta) options.AddSelectorArg(apply, so) options.AddStrictArg(apply, sto) options.AddBuildOptions(apply, bo) diff --git a/pkg/commands/create.go b/pkg/commands/create.go index e46bf1bc99..bee22203ca 100644 --- a/pkg/commands/create.go +++ b/pkg/commands/create.go @@ -30,10 +30,8 @@ import ( // addCreate augments our CLI surface with apply. func addCreate(topLevel *cobra.Command) { koCreateFlags := []string{} - lo := &options.LocalOptions{} - no := &options.NameOptions{} + po := &options.PublishOptions{} fo := &options.FilenameOptions{} - ta := &options.TagsOptions{} so := &options.SelectorOptions{} sto := &options.StrictOptions{} bo := &options.BuildOptions{} @@ -75,7 +73,7 @@ func addCreate(topLevel *cobra.Command) { if err != nil { log.Fatalf("error creating builder: %v", err) } - publisher, err := makePublisher(no, lo, ta) + publisher, err := makePublisher(po) if err != nil { log.Fatalf("error creating publisher: %v", err) } @@ -145,10 +143,8 @@ func addCreate(topLevel *cobra.Command) { } }, } - options.AddLocalArg(create, lo) - options.AddNamingArgs(create, no) + options.AddPublishArg(create, po) options.AddFileArg(create, fo) - options.AddTagsArg(create, ta) options.AddSelectorArg(create, so) options.AddStrictArg(create, sto) options.AddBuildOptions(create, bo) diff --git a/pkg/commands/options/binary.go b/pkg/commands/options/binary.go deleted file mode 100644 index c44a32fa5a..0000000000 --- a/pkg/commands/options/binary.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package options - -import ( - "github.com/spf13/cobra" -) - -// PublishOptions represents options for the ko binary. -type PublishOptions struct { - // Path is the import path of the binary to publish. - Path string -} - -func AddImageArg(cmd *cobra.Command, lo *PublishOptions) { - cmd.Flags().StringVarP(&lo.Path, "image", "i", lo.Path, - "The import path of the binary to publish.") -} diff --git a/pkg/commands/options/local.go b/pkg/commands/options/local.go deleted file mode 100644 index 967c8926e5..0000000000 --- a/pkg/commands/options/local.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package options - -import ( - "github.com/spf13/cobra" -) - -// LocalOptions represents options for the ko binary. -type LocalOptions struct { - // Local publishes images to a local docker daemon. - Local bool - InsecureRegistry bool -} - -func AddLocalArg(cmd *cobra.Command, lo *LocalOptions) { - cmd.Flags().BoolVarP(&lo.Local, "local", "L", lo.Local, - "Whether to publish images to a local docker daemon vs. a registry.") - cmd.Flags().BoolVar(&lo.InsecureRegistry, "insecure-registry", lo.InsecureRegistry, - "Whether to skip TLS verification on the registry") -} diff --git a/pkg/commands/options/flatname.go b/pkg/commands/options/publish.go similarity index 60% rename from pkg/commands/options/flatname.go rename to pkg/commands/options/publish.go index 98574750d9..12b748d3e6 100644 --- a/pkg/commands/options/flatname.go +++ b/pkg/commands/options/publish.go @@ -23,18 +23,32 @@ import ( "github.com/spf13/cobra" ) -// NameOptions represents options for the ko binary. -type NameOptions struct { +// PublishOptions encapsulates options when publishing. +type PublishOptions struct { + Tags []string + + // Local publishes images to a local docker daemon. + Local bool + InsecureRegistry bool + // PreserveImportPaths preserves the full import path after KO_DOCKER_REPO. PreserveImportPaths bool // BaseImportPaths uses the base path without MD5 hash after KO_DOCKER_REPO. BaseImportPaths bool } -func AddNamingArgs(cmd *cobra.Command, no *NameOptions) { - cmd.Flags().BoolVarP(&no.PreserveImportPaths, "preserve-import-paths", "P", no.PreserveImportPaths, +func AddPublishArg(cmd *cobra.Command, po *PublishOptions) { + cmd.Flags().StringSliceVarP(&po.Tags, "tags", "t", []string{"latest"}, + "Which tags to use for the produced image instead of the default 'latest' tag.") + + cmd.Flags().BoolVarP(&po.Local, "local", "L", po.Local, + "Whether to publish images to a local docker daemon vs. a registry.") + cmd.Flags().BoolVar(&po.InsecureRegistry, "insecure-registry", po.InsecureRegistry, + "Whether to skip TLS verification on the registry") + + cmd.Flags().BoolVarP(&po.PreserveImportPaths, "preserve-import-paths", "P", po.PreserveImportPaths, "Whether to preserve the full import path after KO_DOCKER_REPO.") - cmd.Flags().BoolVarP(&no.BaseImportPaths, "base-import-paths", "B", no.BaseImportPaths, + cmd.Flags().BoolVarP(&po.BaseImportPaths, "base-import-paths", "B", po.BaseImportPaths, "Whether to use the base path without MD5 hash after KO_DOCKER_REPO.") } @@ -52,10 +66,10 @@ func baseImportPaths(importpath string) string { return filepath.Base(importpath) } -func MakeNamer(no *NameOptions) publish.Namer { - if no.PreserveImportPaths { +func MakeNamer(po *PublishOptions) publish.Namer { + if po.PreserveImportPaths { return preserveImportPath - } else if no.BaseImportPaths { + } else if po.BaseImportPaths { return baseImportPaths } return packageWithMD5 diff --git a/pkg/commands/options/tags.go b/pkg/commands/options/tags.go deleted file mode 100644 index 22c1d98081..0000000000 --- a/pkg/commands/options/tags.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2018 Google LLC All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package options - -import ( - "github.com/spf13/cobra" -) - -// TagsOptions holds the list of tags to tag the built image -type TagsOptions struct { - Tags []string -} - -func AddTagsArg(cmd *cobra.Command, ta *TagsOptions) { - cmd.Flags().StringSliceVarP(&ta.Tags, "tags", "t", []string{"latest"}, - "Which tags to use for the produced image instead of the default 'latest' tag.") -} diff --git a/pkg/commands/publish.go b/pkg/commands/publish.go index 7a1280c9ba..4a79acc03a 100644 --- a/pkg/commands/publish.go +++ b/pkg/commands/publish.go @@ -24,9 +24,7 @@ import ( // addPublish augments our CLI surface with publish. func addPublish(topLevel *cobra.Command) { - lo := &options.LocalOptions{} - no := &options.NameOptions{} - ta := &options.TagsOptions{} + po := &options.PublishOptions{} bo := &options.BuildOptions{} publish := &cobra.Command{ @@ -64,7 +62,7 @@ func addPublish(topLevel *cobra.Command) { if err != nil { log.Fatalf("error creating builder: %v", err) } - publisher, err := makePublisher(no, lo, ta) + publisher, err := makePublisher(po) if err != nil { log.Fatalf("error creating publisher: %v", err) } @@ -78,9 +76,7 @@ func addPublish(topLevel *cobra.Command) { } }, } - options.AddLocalArg(publish, lo) - options.AddNamingArgs(publish, no) - options.AddTagsArg(publish, ta) + options.AddPublishArg(publish, po) options.AddBuildOptions(publish, bo) topLevel.AddCommand(publish) } diff --git a/pkg/commands/resolve.go b/pkg/commands/resolve.go index 71ac1f2e5d..60ad6f5a94 100644 --- a/pkg/commands/resolve.go +++ b/pkg/commands/resolve.go @@ -24,10 +24,8 @@ import ( // addResolve augments our CLI surface with resolve. func addResolve(topLevel *cobra.Command) { - lo := &options.LocalOptions{} - no := &options.NameOptions{} + po := &options.PublishOptions{} fo := &options.FilenameOptions{} - ta := &options.TagsOptions{} so := &options.SelectorOptions{} sto := &options.StrictOptions{} bo := &options.BuildOptions{} @@ -62,7 +60,7 @@ func addResolve(topLevel *cobra.Command) { if err != nil { log.Fatalf("error creating builder: %v", err) } - publisher, err := makePublisher(no, lo, ta) + publisher, err := makePublisher(po) if err != nil { log.Fatalf("error creating publisher: %v", err) } @@ -72,10 +70,8 @@ func addResolve(topLevel *cobra.Command) { } }, } - options.AddLocalArg(resolve, lo) - options.AddNamingArgs(resolve, no) + options.AddPublishArg(resolve, po) options.AddFileArg(resolve, fo) - options.AddTagsArg(resolve, ta) options.AddSelectorArg(resolve, so) options.AddStrictArg(resolve, sto) options.AddBuildOptions(resolve, bo) diff --git a/pkg/commands/resolver.go b/pkg/commands/resolver.go index 342961e7fc..bf4c38240c 100644 --- a/pkg/commands/resolver.go +++ b/pkg/commands/resolver.go @@ -111,15 +111,15 @@ func makeBuilder(bo *options.BuildOptions) (*build.Caching, error) { return build.NewCaching(innerBuilder) } -func makePublisher(no *options.NameOptions, lo *options.LocalOptions, ta *options.TagsOptions) (publish.Interface, error) { +func makePublisher(po *options.PublishOptions) (publish.Interface, error) { // Create the publish.Interface that we will use to publish image references // to either a docker daemon or a container image registry. innerPublisher, err := func() (publish.Interface, error) { - namer := options.MakeNamer(no) + namer := options.MakeNamer(po) repoName := os.Getenv("KO_DOCKER_REPO") - if lo.Local || repoName == publish.LocalDomain { - return publish.NewDaemon(namer, ta.Tags), nil + if po.Local || repoName == publish.LocalDomain { + return publish.NewDaemon(namer, po.Tags), nil } if repoName == "" { return nil, errors.New("KO_DOCKER_REPO environment variable is unset") @@ -134,8 +134,8 @@ func makePublisher(no *options.NameOptions, lo *options.LocalOptions, ta *option publish.WithTransport(defaultTransport()), publish.WithAuthFromKeychain(authn.DefaultKeychain), publish.WithNamer(namer), - publish.WithTags(ta.Tags), - publish.Insecure(lo.InsecureRegistry)) + publish.WithTags(po.Tags), + publish.Insecure(po.InsecureRegistry)) }() if err != nil { return nil, err diff --git a/pkg/commands/run.go b/pkg/commands/run.go index 184fd19376..4f2f1f2eb2 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -27,9 +27,7 @@ import ( // addRun augments our CLI surface with run. func addRun(topLevel *cobra.Command) { - lo := &options.LocalOptions{} - no := &options.NameOptions{} - ta := &options.TagsOptions{} + po := &options.PublishOptions{} bo := &options.BuildOptions{} run := &cobra.Command{ @@ -69,7 +67,7 @@ func addRun(topLevel *cobra.Command) { if err != nil { log.Fatalf("error creating builder: %v", err) } - publisher, err := makePublisher(no, lo, ta) + publisher, err := makePublisher(po) if err != nil { log.Fatalf("error creating publisher: %v", err) } @@ -137,9 +135,7 @@ func addRun(topLevel *cobra.Command) { UnknownFlags: true, }, } - options.AddLocalArg(run, lo) - options.AddNamingArgs(run, no) - options.AddTagsArg(run, ta) + options.AddPublishArg(run, po) options.AddBuildOptions(run, bo) topLevel.AddCommand(run) From 12df8961be8f1783c810d5a7788afcc0b7d20a4e Mon Sep 17 00:00:00 2001 From: Jon Johnson Date: Wed, 12 Feb 2020 15:52:15 -0800 Subject: [PATCH 4/7] Add options for tarball/layout Adds --oci-layout-path, --tarball, and --push flags. --push=false will disable the default behavior of publishing to a registry. --- pkg/commands/options/publish.go | 13 ++++++++++- pkg/commands/resolver.go | 40 +++++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/pkg/commands/options/publish.go b/pkg/commands/options/publish.go index 12b748d3e6..2e8b4253cb 100644 --- a/pkg/commands/options/publish.go +++ b/pkg/commands/options/publish.go @@ -27,10 +27,16 @@ import ( type PublishOptions struct { Tags []string + // Push publishes images to a registry. + Push bool + // Local publishes images to a local docker daemon. Local bool InsecureRegistry bool + OCILayoutPath string + TarballFile string + // PreserveImportPaths preserves the full import path after KO_DOCKER_REPO. PreserveImportPaths bool // BaseImportPaths uses the base path without MD5 hash after KO_DOCKER_REPO. @@ -41,11 +47,16 @@ func AddPublishArg(cmd *cobra.Command, po *PublishOptions) { cmd.Flags().StringSliceVarP(&po.Tags, "tags", "t", []string{"latest"}, "Which tags to use for the produced image instead of the default 'latest' tag.") + cmd.Flags().BoolVar(&po.Push, "push", true, "Push images to KO_DOCKER_REPO") + cmd.Flags().BoolVarP(&po.Local, "local", "L", po.Local, - "Whether to publish images to a local docker daemon vs. a registry.") + "Load into images to local docker daemon.") cmd.Flags().BoolVar(&po.InsecureRegistry, "insecure-registry", po.InsecureRegistry, "Whether to skip TLS verification on the registry") + cmd.Flags().StringVar(&po.OCILayoutPath, "oci-layout-path", "", "Path to save the OCI image layout of the built images") + cmd.Flags().StringVar(&po.TarballFile, "tarball", "", "File to save images tarballs") + cmd.Flags().BoolVarP(&po.PreserveImportPaths, "preserve-import-paths", "P", po.PreserveImportPaths, "Whether to preserve the full import path after KO_DOCKER_REPO.") cmd.Flags().BoolVarP(&po.BaseImportPaths, "base-import-paths", "B", po.BaseImportPaths, diff --git a/pkg/commands/resolver.go b/pkg/commands/resolver.go index bf4c38240c..49502e8f9c 100644 --- a/pkg/commands/resolver.go +++ b/pkg/commands/resolver.go @@ -115,12 +115,15 @@ func makePublisher(po *options.PublishOptions) (publish.Interface, error) { // Create the publish.Interface that we will use to publish image references // to either a docker daemon or a container image registry. innerPublisher, err := func() (publish.Interface, error) { - namer := options.MakeNamer(po) - repoName := os.Getenv("KO_DOCKER_REPO") - if po.Local || repoName == publish.LocalDomain { + namer := options.MakeNamer(po) + if repoName == publish.LocalDomain || po.Local { + // TODO(jonjohnsonjr): I'm assuming that nobody will + // use local with other publishers, but that might + // not be true. return publish.NewDaemon(namer, po.Tags), nil } + if repoName == "" { return nil, errors.New("KO_DOCKER_REPO environment variable is unset") } @@ -130,12 +133,31 @@ func makePublisher(po *options.PublishOptions) (publish.Interface, error) { } } - return publish.NewDefault(repoName, - publish.WithTransport(defaultTransport()), - publish.WithAuthFromKeychain(authn.DefaultKeychain), - publish.WithNamer(namer), - publish.WithTags(po.Tags), - publish.Insecure(po.InsecureRegistry)) + publishers := []publish.Interface{} + if po.OCILayoutPath != "" { + lp, err := publish.NewLayout(po.OCILayoutPath) + if err != nil { + return nil, fmt.Errorf("failed to create LayoutPublisher for %q: %v", po.OCILayoutPath, err) + } + publishers = append(publishers, lp) + } + if po.TarballFile != "" { + tp := publish.NewTarball(po.TarballFile, repoName, namer, po.Tags) + publishers = append(publishers, tp) + } + if po.Push { + dp, err := publish.NewDefault(repoName, + publish.WithTransport(defaultTransport()), + publish.WithAuthFromKeychain(authn.DefaultKeychain), + publish.WithNamer(namer), + publish.WithTags(po.Tags), + publish.Insecure(po.InsecureRegistry)) + if err != nil { + return nil, err + } + publishers = append(publishers, dp) + } + return publish.MultiPublisher(publishers...), nil }() if err != nil { return nil, err From 8129e5ab9f637f884f12c3c54b4a891bfe733c19 Mon Sep 17 00:00:00 2001 From: Jon Johnson Date: Thu, 13 Feb 2020 09:46:01 -0800 Subject: [PATCH 5/7] go mod vendor --- .../pkg/v1/layout/blob.go | 38 +++ .../go-containerregistry/pkg/v1/layout/doc.go | 19 ++ .../pkg/v1/layout/image.go | 131 ++++++++ .../pkg/v1/layout/index.go | 153 +++++++++ .../pkg/v1/layout/layoutpath.go | 25 ++ .../pkg/v1/layout/options.go | 42 +++ .../pkg/v1/layout/read.go | 32 ++ .../pkg/v1/layout/write.go | 301 ++++++++++++++++++ vendor/github.com/sirupsen/logrus/go.mod | 2 - vendor/modules.txt | 1 + 10 files changed, 742 insertions(+), 2 deletions(-) create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go new file mode 100644 index 0000000000..ba90d4cdb4 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/blob.go @@ -0,0 +1,38 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "io" + "io/ioutil" + "os" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +// Blob returns a blob with the given hash from the Path. +func (l Path) Blob(h v1.Hash) (io.ReadCloser, error) { + return os.Open(l.blobPath(h)) +} + +// Bytes is a convenience function to return a blob from the Path as +// a byte slice. +func (l Path) Bytes(h v1.Hash) ([]byte, error) { + return ioutil.ReadFile(l.blobPath(h)) +} + +func (l Path) blobPath(h v1.Hash) string { + return l.path("blobs", h.Algorithm, h.Hex) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go new file mode 100644 index 0000000000..d80d273639 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/doc.go @@ -0,0 +1,19 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout provides facilities for reading/writing artifacts from/to +// an OCI image layout on disk, see: +// +// https://github.com/opencontainers/image-spec/blob/master/image-layout.md +package layout diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go new file mode 100644 index 0000000000..7c76a10cb3 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/image.go @@ -0,0 +1,131 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "fmt" + "io" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type layoutImage struct { + path Path + desc v1.Descriptor + manifestLock sync.Mutex // Protects rawManifest + rawManifest []byte +} + +var _ partial.CompressedImageCore = (*layoutImage)(nil) + +// Image reads a v1.Image with digest h from the Path. +func (l Path) Image(h v1.Hash) (v1.Image, error) { + ii, err := l.ImageIndex() + if err != nil { + return nil, err + } + + return ii.Image(h) +} + +func (li *layoutImage) MediaType() (types.MediaType, error) { + return li.desc.MediaType, nil +} + +// Implements WithManifest for partial.Blobset. +func (li *layoutImage) Manifest() (*v1.Manifest, error) { + return partial.Manifest(li) +} + +func (li *layoutImage) RawManifest() ([]byte, error) { + li.manifestLock.Lock() + defer li.manifestLock.Unlock() + if li.rawManifest != nil { + return li.rawManifest, nil + } + + b, err := li.path.Bytes(li.desc.Digest) + if err != nil { + return nil, err + } + + li.rawManifest = b + return li.rawManifest, nil +} + +func (li *layoutImage) RawConfigFile() ([]byte, error) { + manifest, err := li.Manifest() + if err != nil { + return nil, err + } + + return li.path.Bytes(manifest.Config.Digest) +} + +func (li *layoutImage) LayerByDigest(h v1.Hash) (partial.CompressedLayer, error) { + manifest, err := li.Manifest() + if err != nil { + return nil, err + } + + if h == manifest.Config.Digest { + return partial.CompressedLayer(&compressedBlob{ + path: li.path, + desc: manifest.Config, + }), nil + } + + for _, desc := range manifest.Layers { + if h == desc.Digest { + switch desc.MediaType { + case types.OCILayer, types.DockerLayer: + return partial.CompressedToLayer(&compressedBlob{ + path: li.path, + desc: desc, + }) + default: + // TODO: We assume everything is a compressed blob, but that might not be true. + // TODO: Handle foreign layers. + return nil, fmt.Errorf("unexpected media type: %v for layer: %v", desc.MediaType, desc.Digest) + } + } + } + + return nil, fmt.Errorf("could not find layer in image: %s", h) +} + +type compressedBlob struct { + path Path + desc v1.Descriptor +} + +func (b *compressedBlob) Digest() (v1.Hash, error) { + return b.desc.Digest, nil +} + +func (b *compressedBlob) Compressed() (io.ReadCloser, error) { + return b.path.Blob(b.desc.Digest) +} + +func (b *compressedBlob) Size() (int64, error) { + return b.desc.Size, nil +} + +func (b *compressedBlob) MediaType() (types.MediaType, error) { + return b.desc.MediaType, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go new file mode 100644 index 0000000000..8ae3a7bb64 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/index.go @@ -0,0 +1,153 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/partial" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +var _ v1.ImageIndex = (*layoutIndex)(nil) + +type layoutIndex struct { + mediaType types.MediaType + path Path + rawIndex []byte +} + +// ImageIndexFromPath is a convenience function which constructs a Path and returns its v1.ImageIndex. +func ImageIndexFromPath(path string) (v1.ImageIndex, error) { + lp, err := FromPath(path) + if err != nil { + return nil, err + } + return lp.ImageIndex() +} + +// ImageIndex returns a v1.ImageIndex for the Path. +func (l Path) ImageIndex() (v1.ImageIndex, error) { + rawIndex, err := ioutil.ReadFile(l.path("index.json")) + if err != nil { + return nil, err + } + + idx := &layoutIndex{ + mediaType: types.OCIImageIndex, + path: l, + rawIndex: rawIndex, + } + + return idx, nil +} + +func (i *layoutIndex) MediaType() (types.MediaType, error) { + return i.mediaType, nil +} + +func (i *layoutIndex) Digest() (v1.Hash, error) { + return partial.Digest(i) +} + +func (i *layoutIndex) Size() (int64, error) { + return partial.Size(i) +} + +func (i *layoutIndex) IndexManifest() (*v1.IndexManifest, error) { + var index v1.IndexManifest + err := json.Unmarshal(i.rawIndex, &index) + return &index, err +} + +func (i *layoutIndex) RawManifest() ([]byte, error) { + return i.rawIndex, nil +} + +func (i *layoutIndex) Image(h v1.Hash) (v1.Image, error) { + // Look up the digest in our manifest first to return a better error. + desc, err := i.findDescriptor(h) + if err != nil { + return nil, err + } + + if !isExpectedMediaType(desc.MediaType, types.OCIManifestSchema1, types.DockerManifestSchema2) { + return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType) + } + + img := &layoutImage{ + path: i.path, + desc: *desc, + } + return partial.CompressedToImage(img) +} + +func (i *layoutIndex) ImageIndex(h v1.Hash) (v1.ImageIndex, error) { + // Look up the digest in our manifest first to return a better error. + desc, err := i.findDescriptor(h) + if err != nil { + return nil, err + } + + if !isExpectedMediaType(desc.MediaType, types.OCIImageIndex, types.DockerManifestList) { + return nil, fmt.Errorf("unexpected media type for %v: %s", h, desc.MediaType) + } + + rawIndex, err := i.path.Bytes(h) + if err != nil { + return nil, err + } + + return &layoutIndex{ + mediaType: desc.MediaType, + path: i.path, + rawIndex: rawIndex, + }, nil +} + +func (i *layoutIndex) Blob(h v1.Hash) (io.ReadCloser, error) { + return i.path.Blob(h) +} + +func (i *layoutIndex) findDescriptor(h v1.Hash) (*v1.Descriptor, error) { + im, err := i.IndexManifest() + if err != nil { + return nil, err + } + + for _, desc := range im.Manifests { + if desc.Digest == h { + return &desc, nil + } + } + + return nil, fmt.Errorf("could not find descriptor in index: %s", h) +} + +// TODO: Pull this out into methods on types.MediaType? e.g. instead, have: +// * mt.IsIndex() +// * mt.IsImage() +func isExpectedMediaType(mt types.MediaType, expected ...types.MediaType) bool { + for _, allowed := range expected { + if mt == allowed { + return true + } + } + return false +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go new file mode 100644 index 0000000000..a031ff5ae9 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/layoutpath.go @@ -0,0 +1,25 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import "path/filepath" + +// Path represents an OCI image layout rooted in a file system path +type Path string + +func (l Path) path(elem ...string) string { + complete := []string{string(l)} + return filepath.Join(append(complete, elem...)...) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go new file mode 100644 index 0000000000..5569e51de3 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/options.go @@ -0,0 +1,42 @@ +package layout + +import v1 "github.com/google/go-containerregistry/pkg/v1" + +// Option is a functional option for Layout. +// +// TODO: We'll need to change this signature to support Sparse/Thin images. +// Or, alternatively, wrap it in a sparse.Image that returns an empty list for layers? +type Option func(*v1.Descriptor) error + +// WithAnnotations adds annotations to the artifact descriptor. +func WithAnnotations(annotations map[string]string) Option { + return func(desc *v1.Descriptor) error { + if desc.Annotations == nil { + desc.Annotations = make(map[string]string) + } + for k, v := range annotations { + desc.Annotations[k] = v + } + + return nil + } +} + +// WithURLs adds urls to the artifact descriptor. +func WithURLs(urls []string) Option { + return func(desc *v1.Descriptor) error { + if desc.URLs == nil { + desc.URLs = []string{} + } + desc.URLs = append(desc.URLs, urls...) + return nil + } +} + +// WithPlatform sets the platform of the artifact descriptor. +func WithPlatform(platform v1.Platform) Option { + return func(desc *v1.Descriptor) error { + desc.Platform = &platform + return nil + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go new file mode 100644 index 0000000000..796abc7dd0 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/read.go @@ -0,0 +1,32 @@ +// Copyright 2019 The original author or authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "os" + "path/filepath" +) + +// FromPath reads an OCI image layout at path and constructs a layout.Path. +func FromPath(path string) (Path, error) { + // TODO: check oci-layout exists + + _, err := os.Stat(filepath.Join(path, "index.json")) + if err != nil { + return "", err + } + + return Path(path), nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go new file mode 100644 index 0000000000..2abb9a586d --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/layout/write.go @@ -0,0 +1,301 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "bytes" + "encoding/json" + "io" + "io/ioutil" + "os" + "path/filepath" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" + "golang.org/x/sync/errgroup" +) + +var layoutFile = `{ + "imageLayoutVersion": "1.0.0" +}` + +// AppendImage writes a v1.Image to the Path and updates +// the index.json to reference it. +func (l Path) AppendImage(img v1.Image, options ...Option) error { + if err := l.writeImage(img); err != nil { + return err + } + + mt, err := img.MediaType() + if err != nil { + return err + } + + d, err := img.Digest() + if err != nil { + return err + } + + manifest, err := img.RawManifest() + if err != nil { + return err + } + + desc := v1.Descriptor{ + MediaType: mt, + Size: int64(len(manifest)), + Digest: d, + } + + for _, opt := range options { + if err := opt(&desc); err != nil { + return err + } + } + + return l.AppendDescriptor(desc) +} + +// AppendIndex writes a v1.ImageIndex to the Path and updates +// the index.json to reference it. +func (l Path) AppendIndex(ii v1.ImageIndex, options ...Option) error { + if err := l.writeIndex(ii); err != nil { + return err + } + + mt, err := ii.MediaType() + if err != nil { + return err + } + + d, err := ii.Digest() + if err != nil { + return err + } + + manifest, err := ii.RawManifest() + if err != nil { + return err + } + + desc := v1.Descriptor{ + MediaType: mt, + Size: int64(len(manifest)), + Digest: d, + } + + for _, opt := range options { + if err := opt(&desc); err != nil { + return err + } + } + + return l.AppendDescriptor(desc) +} + +// AppendDescriptor adds a descriptor to the index.json of the Path. +func (l Path) AppendDescriptor(desc v1.Descriptor) error { + ii, err := l.ImageIndex() + if err != nil { + return err + } + + index, err := ii.IndexManifest() + if err != nil { + return err + } + + index.Manifests = append(index.Manifests, desc) + + rawIndex, err := json.MarshalIndent(index, "", " ") + if err != nil { + return err + } + + return l.writeFile("index.json", rawIndex) +} + +func (l Path) writeFile(name string, data []byte) error { + if err := os.MkdirAll(l.path(), os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + return ioutil.WriteFile(l.path(name), data, os.ModePerm) + +} + +// WriteBlob copies a file to the blobs/ directory in the Path from the given ReadCloser at +// blobs/{hash.Algorithm}/{hash.Hex}. +func (l Path) WriteBlob(hash v1.Hash, r io.ReadCloser) error { + dir := l.path("blobs", hash.Algorithm) + if err := os.MkdirAll(dir, os.ModePerm); err != nil && !os.IsExist(err) { + return err + } + + file := filepath.Join(dir, hash.Hex) + if _, err := os.Stat(file); err == nil { + // Blob already exists, that's fine. + return nil + } + w, err := os.Create(file) + if err != nil { + return err + } + defer w.Close() + + _, err = io.Copy(w, r) + return err +} + +// TODO: A streaming version of WriteBlob so we don't have to know the hash +// before we write it. + +// TODO: For streaming layers we should write to a tmp file then Rename to the +// final digest. +func (l Path) writeLayer(layer v1.Layer) error { + d, err := layer.Digest() + if err != nil { + return err + } + + r, err := layer.Compressed() + if err != nil { + return err + } + + return l.WriteBlob(d, r) +} + +func (l Path) writeImage(img v1.Image) error { + layers, err := img.Layers() + if err != nil { + return err + } + + // Write the layers concurrently. + var g errgroup.Group + for _, layer := range layers { + layer := layer + g.Go(func() error { + return l.writeLayer(layer) + }) + } + if err := g.Wait(); err != nil { + return err + } + + // Write the config. + cfgName, err := img.ConfigName() + if err != nil { + return err + } + cfgBlob, err := img.RawConfigFile() + if err != nil { + return err + } + if err := l.WriteBlob(cfgName, ioutil.NopCloser(bytes.NewReader(cfgBlob))); err != nil { + return err + } + + // Write the img manifest. + d, err := img.Digest() + if err != nil { + return err + } + manifest, err := img.RawManifest() + if err != nil { + return err + } + + return l.WriteBlob(d, ioutil.NopCloser(bytes.NewReader(manifest))) +} + +func (l Path) writeIndexToFile(indexFile string, ii v1.ImageIndex) error { + index, err := ii.IndexManifest() + if err != nil { + return err + } + + // Walk the descriptors and write any v1.Image or v1.ImageIndex that we find. + // If we come across something we don't expect, just write it as a blob. + for _, desc := range index.Manifests { + switch desc.MediaType { + case types.OCIImageIndex, types.DockerManifestList: + ii, err := ii.ImageIndex(desc.Digest) + if err != nil { + return err + } + if err := l.writeIndex(ii); err != nil { + return err + } + case types.OCIManifestSchema1, types.DockerManifestSchema2: + img, err := ii.Image(desc.Digest) + if err != nil { + return err + } + if err := l.writeImage(img); err != nil { + return err + } + default: + // TODO: The layout could reference arbitrary things, which we should + // probably just pass through. + } + } + + rawIndex, err := ii.RawManifest() + if err != nil { + return err + } + + return l.writeFile(indexFile, rawIndex) +} + +func (l Path) writeIndex(ii v1.ImageIndex) error { + // Always just write oci-layout file, since it's small. + if err := l.writeFile("oci-layout", []byte(layoutFile)); err != nil { + return err + } + + h, err := ii.Digest() + if err != nil { + return err + } + + indexFile := filepath.Join("blobs", h.Algorithm, h.Hex) + return l.writeIndexToFile(indexFile, ii) + +} + +// Write constructs a Path at path from an ImageIndex. +// +// The contents are written in the following format: +// At the top level, there is: +// One oci-layout file containing the version of this image-layout. +// One index.json file listing descriptors for the contained images. +// Under blobs/, there is, for each image: +// One file for each layer, named after the layer's SHA. +// One file for each config blob, named after its SHA. +// One file for each manifest blob, named after its SHA. +func Write(path string, ii v1.ImageIndex) (Path, error) { + lp := Path(path) + // Always just write oci-layout file, since it's small. + if err := lp.writeFile("oci-layout", []byte(layoutFile)); err != nil { + return "", err + } + + // TODO create blobs/ in case there is a blobs file which would prevent the directory from being created + + return lp, lp.writeIndexToFile("index.json", ii) +} diff --git a/vendor/github.com/sirupsen/logrus/go.mod b/vendor/github.com/sirupsen/logrus/go.mod index ea2662260e..12fdf98984 100644 --- a/vendor/github.com/sirupsen/logrus/go.mod +++ b/vendor/github.com/sirupsen/logrus/go.mod @@ -8,5 +8,3 @@ require ( github.com/stretchr/testify v1.2.2 golang.org/x/sys v0.0.0-20190422165155-953cdadca894 ) - -go 1.13 diff --git a/vendor/modules.txt b/vendor/modules.txt index eeba453dd2..2e69b37fc1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -93,6 +93,7 @@ github.com/google/go-containerregistry/pkg/name github.com/google/go-containerregistry/pkg/v1 github.com/google/go-containerregistry/pkg/v1/daemon github.com/google/go-containerregistry/pkg/v1/empty +github.com/google/go-containerregistry/pkg/v1/layout github.com/google/go-containerregistry/pkg/v1/mutate github.com/google/go-containerregistry/pkg/v1/partial github.com/google/go-containerregistry/pkg/v1/random From 50a710baf27ac253c25668dbf34964134ec29252 Mon Sep 17 00:00:00 2001 From: Jon Johnson Date: Fri, 14 Feb 2020 10:22:10 -0800 Subject: [PATCH 6/7] Add Close method to publish.Interface This allows us to defer writing to the tarball until we've collected all the images that have been published. --- pkg/commands/apply.go | 1 + pkg/commands/create.go | 1 + pkg/commands/publish.go | 1 + pkg/commands/resolve.go | 1 + pkg/commands/run.go | 1 + pkg/publish/daemon.go | 4 ++++ pkg/publish/default.go | 4 ++++ pkg/publish/layout.go | 4 ++++ pkg/publish/multi.go | 9 +++++++++ pkg/publish/publish.go | 4 ++++ pkg/publish/shared.go | 4 ++++ pkg/publish/tarball.go | 31 +++++++++++++++++++++---------- 12 files changed, 55 insertions(+), 10 deletions(-) diff --git a/pkg/commands/apply.go b/pkg/commands/apply.go index 323823958f..d3f98ac616 100644 --- a/pkg/commands/apply.go +++ b/pkg/commands/apply.go @@ -77,6 +77,7 @@ func addApply(topLevel *cobra.Command) { if err != nil { log.Fatalf("error creating publisher: %v", err) } + defer publisher.Close() // Create a set of ko-specific flags to ignore when passing through // kubectl global flags. ignoreSet := make(map[string]struct{}) diff --git a/pkg/commands/create.go b/pkg/commands/create.go index bee22203ca..1bd0350c09 100644 --- a/pkg/commands/create.go +++ b/pkg/commands/create.go @@ -77,6 +77,7 @@ func addCreate(topLevel *cobra.Command) { if err != nil { log.Fatalf("error creating publisher: %v", err) } + defer publisher.Close() // Create a set of ko-specific flags to ignore when passing through // kubectl global flags. ignoreSet := make(map[string]struct{}) diff --git a/pkg/commands/publish.go b/pkg/commands/publish.go index 4a79acc03a..2c651a1ce2 100644 --- a/pkg/commands/publish.go +++ b/pkg/commands/publish.go @@ -66,6 +66,7 @@ func addPublish(topLevel *cobra.Command) { if err != nil { log.Fatalf("error creating publisher: %v", err) } + defer publisher.Close() ctx := createCancellableContext() images, err := publishImages(ctx, args, publisher, builder) if err != nil { diff --git a/pkg/commands/resolve.go b/pkg/commands/resolve.go index 60ad6f5a94..485d255b9f 100644 --- a/pkg/commands/resolve.go +++ b/pkg/commands/resolve.go @@ -64,6 +64,7 @@ func addResolve(topLevel *cobra.Command) { if err != nil { log.Fatalf("error creating publisher: %v", err) } + defer publisher.Close() ctx := createCancellableContext() if err := resolveFilesToWriter(ctx, builder, publisher, fo, so, sto, os.Stdout); err != nil { log.Fatal(err) diff --git a/pkg/commands/run.go b/pkg/commands/run.go index 4f2f1f2eb2..2b43de2680 100644 --- a/pkg/commands/run.go +++ b/pkg/commands/run.go @@ -71,6 +71,7 @@ func addRun(topLevel *cobra.Command) { if err != nil { log.Fatalf("error creating publisher: %v", err) } + defer publisher.Close() if len(os.Args) < 3 { log.Fatalf("usage: %s run ", os.Args[0]) diff --git a/pkg/publish/daemon.go b/pkg/publish/daemon.go index c8019bbc2c..00cf0f9a63 100644 --- a/pkg/publish/daemon.go +++ b/pkg/publish/daemon.go @@ -78,3 +78,7 @@ func (d *demon) Publish(img v1.Image, s string) (name.Reference, error) { return &digestTag, nil } + +func (d *demon) Close() error { + return nil +} diff --git a/pkg/publish/default.go b/pkg/publish/default.go index 8d5cf33c83..23604b00a6 100644 --- a/pkg/publish/default.go +++ b/pkg/publish/default.go @@ -140,3 +140,7 @@ func (d *defalt) Publish(img v1.Image, s string) (name.Reference, error) { log.Printf("Published %v", dig) return &dig, nil } + +func (d *defalt) Close() error { + return nil +} diff --git a/pkg/publish/layout.go b/pkg/publish/layout.go index a216f27a4d..6ea15966d5 100644 --- a/pkg/publish/layout.go +++ b/pkg/publish/layout.go @@ -60,3 +60,7 @@ func (l *LayoutPublisher) Publish(img v1.Image, s string) (name.Reference, error return dig, nil } + +func (l *LayoutPublisher) Close() error { + return nil +} diff --git a/pkg/publish/multi.go b/pkg/publish/multi.go index 4ffa8d7744..d808a06220 100644 --- a/pkg/publish/multi.go +++ b/pkg/publish/multi.go @@ -43,3 +43,12 @@ func (p *multiPublisher) Publish(img v1.Image, s string) (ref name.Reference, er return } + +func (p *multiPublisher) Close() (err error) { + for _, pub := range p.publishers { + if perr := pub.Close(); perr != nil { + err = perr + } + } + return +} diff --git a/pkg/publish/publish.go b/pkg/publish/publish.go index c6002ef950..1c61693e5b 100644 --- a/pkg/publish/publish.go +++ b/pkg/publish/publish.go @@ -25,4 +25,8 @@ type Interface interface { // provided string into the image's repository name. Returns the digest // of the published image. Publish(v1.Image, string) (name.Reference, error) + + // Close exists for the tarball implementation so we can + // do the whole thing in one write. + Close() error } diff --git a/pkg/publish/shared.go b/pkg/publish/shared.go index edd2832b87..1a2af98fcb 100644 --- a/pkg/publish/shared.go +++ b/pkg/publish/shared.go @@ -74,3 +74,7 @@ func (c *caching) Publish(img v1.Image, ref string) (name.Reference, error) { return f.Get() } + +func (c *caching) Close() error { + return c.inner.Close() +} diff --git a/pkg/publish/tarball.go b/pkg/publish/tarball.go index 37895eb26e..17ba8dde68 100644 --- a/pkg/publish/tarball.go +++ b/pkg/publish/tarball.go @@ -29,11 +29,18 @@ type TarballPublisher struct { base string namer Namer tags []string + refs map[name.Reference]v1.Image } // NewTarball returns a new publish.Interface that saves images to a tarball. func NewTarball(file, base string, namer Namer, tags []string) *TarballPublisher { - return &TarballPublisher{file, base, namer, tags} + return &TarballPublisher{ + file: file, + base: base, + namer: namer, + tags: tags, + refs: make(map[name.Reference]v1.Image), + } } // Publish implements publish.Interface. @@ -41,13 +48,12 @@ func (t *TarballPublisher) Publish(img v1.Image, s string) (name.Reference, erro // https://github.com/google/go-containerregistry/issues/212 s = strings.ToLower(s) - refs := make(map[name.Reference]v1.Image) for _, tagName := range t.tags { tag, err := name.ParseReference(fmt.Sprintf("%s/%s:%s", t.base, t.namer(s), tagName)) if err != nil { return nil, err } - refs[tag] = img + t.refs[tag] = img } h, err := img.Digest() @@ -60,7 +66,7 @@ func (t *TarballPublisher) Publish(img v1.Image, s string) (name.Reference, erro if err != nil { return nil, err } - refs[ref] = img + t.refs[ref] = img } ref := fmt.Sprintf("%s/%s@%s", t.base, t.namer(s), h) @@ -74,11 +80,16 @@ func (t *TarballPublisher) Publish(img v1.Image, s string) (name.Reference, erro return nil, err } - log.Printf("Saving %v", dig) - if err := tarball.MultiRefWriteToFile(t.file, refs); err != nil { - return nil, err - } - log.Printf("Saved %v", dig) - return &dig, nil } + +func (t *TarballPublisher) Close() error { + log.Printf("Saving %v", t.file) + if err := tarball.MultiRefWriteToFile(t.file, t.refs); err != nil { + // Bad practice, but we log this here because right now we just defer the Close. + log.Printf("failed to save %q: %v", t.file, err) + return err + } + log.Printf("Saved %v", t.file) + return nil +} From 51ef078aa7580400113726e12794a85ae6103fa8 Mon Sep 17 00:00:00 2001 From: Jon Johnson Date: Fri, 14 Feb 2020 10:35:01 -0800 Subject: [PATCH 7/7] Fix tests --- pkg/internal/testing/fixed.go | 4 ++++ pkg/publish/multi_test.go | 4 ++++ pkg/publish/shared_test.go | 12 ++++++++---- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/pkg/internal/testing/fixed.go b/pkg/internal/testing/fixed.go index 879a46b7c5..472b0bb949 100644 --- a/pkg/internal/testing/fixed.go +++ b/pkg/internal/testing/fixed.go @@ -72,6 +72,10 @@ func (f *fixedPublish) Publish(_ v1.Image, s string) (name.Reference, error) { return &d, nil } +func (f *fixedPublish) Close() error { + return nil +} + func ComputeDigest(base name.Repository, ref string, h v1.Hash) string { d, err := name.NewDigest(fmt.Sprintf("%s/%s@%s", base, ref, h)) if err != nil { diff --git a/pkg/publish/multi_test.go b/pkg/publish/multi_test.go index 99d4207920..46ed21205a 100644 --- a/pkg/publish/multi_test.go +++ b/pkg/publish/multi_test.go @@ -56,4 +56,8 @@ func TestMulti(t *testing.T) { if _, err := p.Publish(img, importpath); err != nil { t.Errorf("Publish() = %v", err) } + + if err := p.Close(); err != nil { + t.Errorf("Close() = %v", err) + } } diff --git a/pkg/publish/shared_test.go b/pkg/publish/shared_test.go index cde0d186f6..858a786b9f 100644 --- a/pkg/publish/shared_test.go +++ b/pkg/publish/shared_test.go @@ -30,17 +30,21 @@ type slowpublish struct { // slowpublish implements Interface var _ Interface = (*slowpublish)(nil) -func (sb *slowpublish) Publish(img v1.Image, ref string) (name.Reference, error) { - time.Sleep(sb.sleep) +func (sp *slowpublish) Publish(img v1.Image, ref string) (name.Reference, error) { + time.Sleep(sp.sleep) return makeRef() } +func (sp *slowpublish) Close() error { + return nil +} + func TestCaching(t *testing.T) { duration := 100 * time.Millisecond ref := "foo" - sb := &slowpublish{duration} - cb, _ := NewCaching(sb) + sp := &slowpublish{duration} + cb, _ := NewCaching(sp) previousDigest := "not-a-digest" // Each iteration, we test that the first publish is slow and subsequent