From de5dc19c43e9a5179dedf441b8463826164bbc99 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Mon, 28 Feb 2022 13:48:27 -0500 Subject: [PATCH 1/5] Produce OCI images by default This changes build logic to prefer to produce OCI images and indexes, even if original base images are Docker manifests or manifest lists. OCI indexes support annotations, while Docker manifest lists do not, and we'd like to inject base image information in annotations wherever possible. Since Quay.io recently added support for OCI manifests, this is no longer a serious breaking change -- and anyway, producing SBOMs by default already breaks Quay.io without --sbom=none. This behavior can be disabled with --preserve-docker-media-type=true, which will result in Docker-type manifests being produced if and only if the base image was a Docker-typed manifest. This partially reverts commit 42723d75e7076c4946351c9e3197ce65ff31b4ec. --- .github/workflows/e2e.yaml | 10 ++++ pkg/build/gobuild.go | 108 +++++++++++++++++++-------------- pkg/build/gobuild_test.go | 109 ++++++++++++++++++++++++++++++++++ pkg/build/options.go | 10 ++++ pkg/commands/options/build.go | 6 ++ pkg/commands/resolver.go | 1 + pkg/publish/default.go | 1 + 7 files changed, 202 insertions(+), 43 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 1885d464c7..28719127fd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -20,6 +20,7 @@ jobs: - uses: actions/setup-go@v2 with: go-version: 1.17.x + - uses: imjasonh/setup-crane@v0.1 - name: Build and run ko container env: @@ -67,3 +68,12 @@ jobs: EOF docker run $(go run ./ build ./test/ --platform=${PLATFORM}) --wait=false 2>&1 | grep "${{ github.sha }}" + # Check that images are produced in OCI formats unless explicitly requested otherwise. + crane manifest $(go run ./ build ./test/) | jq -r '.mediaType' | \ + grep "application/vnd.oci.image.manifest.v1+json" + crane manifest $(go run ./ build ./test/ --preserve-docker-media-type) | jq -r '.mediaType' | \ + grep "application/vnd.docker.distribution.manifest.v2+json" + crane manifest $(go run ./ build ./test/ --platform=all) | jq -r '.mediaType' | \ + grep "application/vnd.oci.image.index.v1+json" + crane manifest $(go run ./ build ./test/ --platform=all --preserve-docker-media-type) | jq -r '.mediaType' | \ + grep "application/vnd.docker.distribution.manifest.list.v2+json" diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go index 9a8af12a47..c971cb408f 100644 --- a/pkg/build/gobuild.go +++ b/pkg/build/gobuild.go @@ -70,19 +70,20 @@ type platformMatcher struct { } type gobuild struct { - ctx context.Context - getBase GetBase - creationTime v1.Time - kodataCreationTime v1.Time - build builder - sbom sbomber - disableOptimizations bool - trimpath bool - buildConfigs map[string]Config - platformMatcher *platformMatcher - dir string - labels map[string]string - semaphore *semaphore.Weighted + ctx context.Context + getBase GetBase + creationTime v1.Time + kodataCreationTime v1.Time + build builder + sbom sbomber + disableOptimizations bool + trimpath bool + preserveDockerMediaType bool + buildConfigs map[string]Config + platformMatcher *platformMatcher + dir string + labels map[string]string + semaphore *semaphore.Weighted cache *layerCache } @@ -91,19 +92,20 @@ type gobuild struct { type Option func(*gobuildOpener) error type gobuildOpener struct { - ctx context.Context - getBase GetBase - creationTime v1.Time - kodataCreationTime v1.Time - build builder - sbom sbomber - disableOptimizations bool - trimpath bool - buildConfigs map[string]Config - platforms []string - labels map[string]string - dir string - jobs int + ctx context.Context + getBase GetBase + creationTime v1.Time + kodataCreationTime v1.Time + build builder + sbom sbomber + disableOptimizations bool + trimpath bool + preserveDockerMediaType bool + buildConfigs map[string]Config + platforms []string + labels map[string]string + dir string + jobs int } func (gbo *gobuildOpener) Open() (Interface, error) { @@ -118,18 +120,19 @@ func (gbo *gobuildOpener) Open() (Interface, error) { gbo.jobs = runtime.GOMAXPROCS(0) } return &gobuild{ - ctx: gbo.ctx, - getBase: gbo.getBase, - creationTime: gbo.creationTime, - kodataCreationTime: gbo.kodataCreationTime, - build: gbo.build, - sbom: gbo.sbom, - disableOptimizations: gbo.disableOptimizations, - trimpath: gbo.trimpath, - buildConfigs: gbo.buildConfigs, - labels: gbo.labels, - dir: gbo.dir, - platformMatcher: matcher, + ctx: gbo.ctx, + getBase: gbo.getBase, + creationTime: gbo.creationTime, + kodataCreationTime: gbo.kodataCreationTime, + build: gbo.build, + sbom: gbo.sbom, + disableOptimizations: gbo.disableOptimizations, + trimpath: gbo.trimpath, + preserveDockerMediaType: gbo.preserveDockerMediaType, + buildConfigs: gbo.buildConfigs, + labels: gbo.labels, + dir: gbo.dir, + platformMatcher: matcher, cache: &layerCache{ buildToDiff: map[string]buildIDToDiffID{}, diffToDesc: map[string]diffIDToDescriptor{}, @@ -694,6 +697,16 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl ref := newRef(refStr) + baseType := types.OCIManifestSchema1 + if g.preserveDockerMediaType { + var err error + baseType, err = base.MediaType() + if err != nil { + return nil, err + } + } + base = mutate.MediaType(base, baseType) + cf, err := base.ConfigFile() if err != nil { return nil, err @@ -884,7 +897,7 @@ func (g *gobuild) Build(ctx context.Context, s string) (Result, error) { // Annotate the base image we pass to the build function with // annotations indicating the digest (and possibly tag) of the // base image. This will be inherited by the image produced. - if mt != types.DockerManifestList { + if mt != types.DockerManifestList && !g.preserveDockerMediaType { anns := map[string]string{ specsv1.AnnotationBaseImageDigest: baseDigest.String(), } @@ -963,11 +976,17 @@ func (g *gobuild) buildAll(ctx context.Context, ref string, baseIndex v1.ImageIn if err != nil { return err } + + mt := types.OCIManifestSchema1 + if g.preserveDockerMediaType { + mt = desc.MediaType + } + adds[i] = ocimutate.IndexAddendum{ Add: img, Descriptor: v1.Descriptor{ URLs: desc.URLs, - MediaType: desc.MediaType, + MediaType: mt, Annotations: desc.Annotations, Platform: desc.Platform, }, @@ -979,9 +998,12 @@ func (g *gobuild) buildAll(ctx context.Context, ref string, baseIndex v1.ImageIn return nil, err } - baseType, err := baseIndex.MediaType() - if err != nil { - return nil, err + baseType := types.OCIImageIndex + if g.preserveDockerMediaType { + baseType, err = baseIndex.MediaType() + if err != nil { + return nil, err + } } idx := ocimutate.AppendManifests(mutate.IndexMediaType(empty.Index, baseType), adds...) diff --git a/pkg/build/gobuild_test.go b/pkg/build/gobuild_test.go index 890b81afb9..0f3035fdd9 100644 --- a/pkg/build/gobuild_test.go +++ b/pkg/build/gobuild_test.go @@ -513,6 +513,17 @@ func TestGoBuildNoKoData(t *testing.T) { t.Errorf("created = %v, want %v", actual, creationTime) } }) + + t.Run("check OCI media type", func(t *testing.T) { + mt, err := img.MediaType() + if err != nil { + t.Errorf("MediaType() = %v", err) + } + + if got, want := mt, types.OCIManifestSchema1; got != want { + t.Errorf("mediaType = %v, want %v", got, want) + } + }) } func validateImage(t *testing.T, img oci.SignedImage, baseLayers int64, creationTime v1.Time, checkAnnotations bool, expectSBOM bool) { @@ -919,6 +930,104 @@ func TestGoBuildIndex(t *testing.T) { t.Errorf("Digest mismatch: %s != %s", d1, d2) } }) + + t.Run("check OCI media type", func(t *testing.T) { + mt, err := idx.MediaType() + if err != nil { + t.Fatalf("MediaType() = %v", err) + } + + if got, want := mt, types.OCIImageIndex; got != want { + t.Errorf("mediaType = %v, want %v", got, want) + } + + for i, mf := range im.Manifests { + if got, want := mf.MediaType, types.OCIManifestSchema1; got != want { + t.Errorf("manifest[%d] mediaType = %s, want %s", i, got, want) + } + } + }) +} + +func TestPreserveDockerMediaType(t *testing.T) { + mustRandomImage := func(t *testing.T) v1.Image { + img, err := random.Image(1, 1) + if err != nil { + t.Fatal(err) + } + return img + } + mustRandomIndex := func(t *testing.T) v1.ImageIndex { + idx, err := random.Index(1, 1, 3) + if err != nil { + t.Fatal(err) + } + return idx + } + + for _, c := range []struct { + desc string + preserve bool + base Result + want types.MediaType + }{{ + desc: "docker image -> oci image", + preserve: false, + base: mustRandomImage(t), + want: types.OCIManifestSchema1, + }, { + desc: "docker index -> oci index", + preserve: false, + base: mustRandomIndex(t), + want: types.OCIImageIndex, + }, { + desc: "docker image, preserved", + preserve: true, + base: mustRandomImage(t), + want: types.DockerManifestSchema2, + }, { + desc: "docker index, preserved", + preserve: true, + base: mutate.IndexMediaType(mustRandomIndex(t), types.DockerManifestList), + want: types.DockerManifestList, + }, { + desc: "oci image", + preserve: true, + base: mutate.MediaType(mustRandomImage(t), types.OCIManifestSchema1), + want: types.OCIManifestSchema1, + }, { + desc: "oci index", + preserve: true, + base: mutate.IndexMediaType(mustRandomIndex(t), types.OCIImageIndex), + want: types.OCIImageIndex, + }} { + t.Run(c.desc, func(t *testing.T) { + importpath := "github.com/google/ko" + ng, err := NewGo( + context.Background(), + "", + WithBaseImages(func(context.Context, string) (name.Reference, Result, error) { return baseRef, c.base, nil }), + WithPlatforms("all"), + WithPreserveDockerMediaType(c.preserve), + withBuilder(writeTempFile), + ) + if err != nil { + t.Fatalf("NewGo() = %v", err) + } + + result, err := ng.Build(context.Background(), StrictScheme+filepath.Join(importpath, "test")) + if err != nil { + t.Fatalf("Build() = %v", err) + } + + got, err := result.MediaType() + if err != nil { + t.Errorf("MediaType() = %v", err) + } else if got != c.want { + t.Errorf("Got %q, want %q", got, c.want) + } + }) + } } func TestNestedIndex(t *testing.T) { diff --git a/pkg/build/options.go b/pkg/build/options.go index e29bca5ba1..3bd02e9940 100644 --- a/pkg/build/options.go +++ b/pkg/build/options.go @@ -73,6 +73,16 @@ func WithTrimpath(v bool) Option { } } +// WithPreserveDockerMediaType is a functional option that controls whether to +// preserve Docker media types from base images. If false, images that are +// produced will use OCI media types. +func WithPreserveDockerMediaType(v bool) Option { + return func(gbo *gobuildOpener) error { + gbo.preserveDockerMediaType = v + return nil + } +} + // WithConfig is a functional option for providing GoReleaser Build influenced // build settings for importpaths. // diff --git a/pkg/commands/options/build.go b/pkg/commands/options/build.go index 4f2fec976b..af9f07f61e 100644 --- a/pkg/commands/options/build.go +++ b/pkg/commands/options/build.go @@ -67,6 +67,9 @@ type BuildOptions struct { // BuildConfigs stores the per-image build config from `.ko.yaml`. BuildConfigs map[string]build.Config + + // If true, don't convert Docker-typed base images to OCI when building. + PreserveDockerMediaType bool } func AddBuildOptions(cmd *cobra.Command, bo *BuildOptions) { @@ -80,6 +83,9 @@ func AddBuildOptions(cmd *cobra.Command, bo *BuildOptions) { "Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]*") cmd.Flags().StringSliceVar(&bo.Labels, "image-label", []string{}, "Which labels (key=value) to add to the image.") + + cmd.Flags().BoolVar(&bo.PreserveDockerMediaType, "preserve-docker-media-type", false, "If false, push images in OCI format regardless of base image format") + bo.Trimpath = true } diff --git a/pkg/commands/resolver.go b/pkg/commands/resolver.go index 91c09fba99..97c4e14abe 100644 --- a/pkg/commands/resolver.go +++ b/pkg/commands/resolver.go @@ -108,6 +108,7 @@ func gobuildOptions(bo *options.BuildOptions) ([]build.Option, error) { opts = append(opts, build.WithSPDX(version())) } opts = append(opts, build.WithTrimpath(bo.Trimpath)) + opts = append(opts, build.WithPreserveDockerMediaType(bo.PreserveDockerMediaType)) for _, lf := range bo.Labels { parts := strings.SplitN(lf, "=", 2) if len(parts) != 2 { diff --git a/pkg/publish/default.go b/pkg/publish/default.go index b0f58cc994..8ea3c611eb 100644 --- a/pkg/publish/default.go +++ b/pkg/publish/default.go @@ -180,6 +180,7 @@ func pushResult(ctx context.Context, tag name.Tag, br build.Result, opt []remote return err } } + return remote.WriteIndex(tag, idx, opt...) case types.OCIManifestSchema1, types.DockerManifestSchema2: img, ok := br.(v1.Image) From a39f119329460f787ee33ce1daed16a891885e39 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Mon, 28 Feb 2022 15:04:54 -0500 Subject: [PATCH 2/5] drop e2e test --- .github/workflows/e2e.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 28719127fd..1885d464c7 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -20,7 +20,6 @@ jobs: - uses: actions/setup-go@v2 with: go-version: 1.17.x - - uses: imjasonh/setup-crane@v0.1 - name: Build and run ko container env: @@ -68,12 +67,3 @@ jobs: EOF docker run $(go run ./ build ./test/ --platform=${PLATFORM}) --wait=false 2>&1 | grep "${{ github.sha }}" - # Check that images are produced in OCI formats unless explicitly requested otherwise. - crane manifest $(go run ./ build ./test/) | jq -r '.mediaType' | \ - grep "application/vnd.oci.image.manifest.v1+json" - crane manifest $(go run ./ build ./test/ --preserve-docker-media-type) | jq -r '.mediaType' | \ - grep "application/vnd.docker.distribution.manifest.v2+json" - crane manifest $(go run ./ build ./test/ --platform=all) | jq -r '.mediaType' | \ - grep "application/vnd.oci.image.index.v1+json" - crane manifest $(go run ./ build ./test/ --platform=all --preserve-docker-media-type) | jq -r '.mediaType' | \ - grep "application/vnd.docker.distribution.manifest.list.v2+json" From 4e377932432135d51f521f4b88f124990398183f Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Mon, 28 Feb 2022 15:07:06 -0500 Subject: [PATCH 3/5] update generated docs --- doc/ko_apply.md | 1 + doc/ko_build.md | 35 ++++++++++++++++++----------------- doc/ko_create.md | 1 + doc/ko_resolve.md | 41 +++++++++++++++++++++-------------------- doc/ko_run.md | 35 ++++++++++++++++++----------------- pkg/publish/default.go | 1 - 6 files changed, 59 insertions(+), 55 deletions(-) diff --git a/doc/ko_apply.md b/doc/ko_apply.md index feeb982268..6f12d2809c 100644 --- a/doc/ko_apply.md +++ b/doc/ko_apply.md @@ -69,6 +69,7 @@ ko apply -f FILENAME [flags] --oci-layout-path string Path to save the OCI image layout of the built images --password string Password for basic authentication to the API server (DEPRECATED) --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* + --preserve-docker-media-type If false, push images in OCI format regardless of base image format -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. --push Push images to KO_DOCKER_REPO (default true) -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. diff --git a/doc/ko_build.md b/doc/ko_build.md index cee5b5f40c..8e02bf502b 100644 --- a/doc/ko_build.md +++ b/doc/ko_build.md @@ -43,23 +43,24 @@ ko build IMPORTPATH... [flags] ### Options ``` - --bare Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags). - -B, --base-import-paths Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags). - --disable-optimizations Disable optimizations when building Go code. Useful when you want to interactively debug the created container. - -h, --help help for build - --image-label strings Which labels (key=value) to add to the image. - --image-refs string Path to file where a list of the published image references will be written. - --insecure-registry Whether to skip TLS verification on the registry - -j, --jobs int The maximum number of concurrent builds (default GOMAXPROCS) - -L, --local Load into images to local docker daemon. - --oci-layout-path string Path to save the OCI image layout of the built images - --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* - -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. - --push Push images to KO_DOCKER_REPO (default true) - --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") - --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. - -t, --tags strings Which tags to use for the produced image instead of the default 'latest' tag (may not work properly with --base-import-paths or --bare). (default [latest]) - --tarball string File to save images tarballs + --bare Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags). + -B, --base-import-paths Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags). + --disable-optimizations Disable optimizations when building Go code. Useful when you want to interactively debug the created container. + -h, --help help for build + --image-label strings Which labels (key=value) to add to the image. + --image-refs string Path to file where a list of the published image references will be written. + --insecure-registry Whether to skip TLS verification on the registry + -j, --jobs int The maximum number of concurrent builds (default GOMAXPROCS) + -L, --local Load into images to local docker daemon. + --oci-layout-path string Path to save the OCI image layout of the built images + --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* + --preserve-docker-media-type If false, push images in OCI format regardless of base image format + -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --push Push images to KO_DOCKER_REPO (default true) + --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") + --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. + -t, --tags strings Which tags to use for the produced image instead of the default 'latest' tag (may not work properly with --base-import-paths or --bare). (default [latest]) + --tarball string File to save images tarballs ``` ### Options inherited from parent commands diff --git a/doc/ko_create.md b/doc/ko_create.md index 06615e59b3..f594aadf02 100644 --- a/doc/ko_create.md +++ b/doc/ko_create.md @@ -69,6 +69,7 @@ ko create -f FILENAME [flags] --oci-layout-path string Path to save the OCI image layout of the built images --password string Password for basic authentication to the API server (DEPRECATED) --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* + --preserve-docker-media-type If false, push images in OCI format regardless of base image format -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. --push Push images to KO_DOCKER_REPO (default true) -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. diff --git a/doc/ko_resolve.md b/doc/ko_resolve.md index 2355229005..78e772694d 100644 --- a/doc/ko_resolve.md +++ b/doc/ko_resolve.md @@ -38,26 +38,27 @@ ko resolve -f FILENAME [flags] ### Options ``` - --bare Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags). - -B, --base-import-paths Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags). - --disable-optimizations Disable optimizations when building Go code. Useful when you want to interactively debug the created container. - -f, --filename strings Filename, directory, or URL to files to use to create the resource - -h, --help help for resolve - --image-label strings Which labels (key=value) to add to the image. - --image-refs string Path to file where a list of the published image references will be written. - --insecure-registry Whether to skip TLS verification on the registry - -j, --jobs int The maximum number of concurrent builds (default GOMAXPROCS) - -L, --local Load into images to local docker daemon. - --oci-layout-path string Path to save the OCI image layout of the built images - --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* - -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. - --push Push images to KO_DOCKER_REPO (default true) - -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. - --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") - -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2) - --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. - -t, --tags strings Which tags to use for the produced image instead of the default 'latest' tag (may not work properly with --base-import-paths or --bare). (default [latest]) - --tarball string File to save images tarballs + --bare Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags). + -B, --base-import-paths Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags). + --disable-optimizations Disable optimizations when building Go code. Useful when you want to interactively debug the created container. + -f, --filename strings Filename, directory, or URL to files to use to create the resource + -h, --help help for resolve + --image-label strings Which labels (key=value) to add to the image. + --image-refs string Path to file where a list of the published image references will be written. + --insecure-registry Whether to skip TLS verification on the registry + -j, --jobs int The maximum number of concurrent builds (default GOMAXPROCS) + -L, --local Load into images to local docker daemon. + --oci-layout-path string Path to save the OCI image layout of the built images + --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* + --preserve-docker-media-type If false, push images in OCI format regardless of base image format + -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --push Push images to KO_DOCKER_REPO (default true) + -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. + --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") + -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2) + --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. + -t, --tags strings Which tags to use for the produced image instead of the default 'latest' tag (may not work properly with --base-import-paths or --bare). (default [latest]) + --tarball string File to save images tarballs ``` ### Options inherited from parent commands diff --git a/doc/ko_run.md b/doc/ko_run.md index 02b08d5923..ebecc7948d 100644 --- a/doc/ko_run.md +++ b/doc/ko_run.md @@ -30,23 +30,24 @@ ko run IMPORTPATH [flags] ### Options ``` - --bare Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags). - -B, --base-import-paths Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags). - --disable-optimizations Disable optimizations when building Go code. Useful when you want to interactively debug the created container. - -h, --help help for run - --image-label strings Which labels (key=value) to add to the image. - --image-refs string Path to file where a list of the published image references will be written. - --insecure-registry Whether to skip TLS verification on the registry - -j, --jobs int The maximum number of concurrent builds (default GOMAXPROCS) - -L, --local Load into images to local docker daemon. - --oci-layout-path string Path to save the OCI image layout of the built images - --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* - -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. - --push Push images to KO_DOCKER_REPO (default true) - --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") - --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. - -t, --tags strings Which tags to use for the produced image instead of the default 'latest' tag (may not work properly with --base-import-paths or --bare). (default [latest]) - --tarball string File to save images tarballs + --bare Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags). + -B, --base-import-paths Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags). + --disable-optimizations Disable optimizations when building Go code. Useful when you want to interactively debug the created container. + -h, --help help for run + --image-label strings Which labels (key=value) to add to the image. + --image-refs string Path to file where a list of the published image references will be written. + --insecure-registry Whether to skip TLS verification on the registry + -j, --jobs int The maximum number of concurrent builds (default GOMAXPROCS) + -L, --local Load into images to local docker daemon. + --oci-layout-path string Path to save the OCI image layout of the built images + --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* + --preserve-docker-media-type If false, push images in OCI format regardless of base image format + -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --push Push images to KO_DOCKER_REPO (default true) + --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") + --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. + -t, --tags strings Which tags to use for the produced image instead of the default 'latest' tag (may not work properly with --base-import-paths or --bare). (default [latest]) + --tarball string File to save images tarballs ``` ### Options inherited from parent commands diff --git a/pkg/publish/default.go b/pkg/publish/default.go index 8ea3c611eb..b0f58cc994 100644 --- a/pkg/publish/default.go +++ b/pkg/publish/default.go @@ -180,7 +180,6 @@ func pushResult(ctx context.Context, tag name.Tag, br build.Result, opt []remote return err } } - return remote.WriteIndex(tag, idx, opt...) case types.OCIManifestSchema1, types.DockerManifestSchema2: img, ok := br.(v1.Image) From 0e88f264563b1f60319f9e1d7ba4c4ed2761b33b Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Fri, 4 Mar 2022 14:06:16 -0500 Subject: [PATCH 4/5] --preserve-media-type --- pkg/build/gobuild.go | 90 +++++++++++++++++------------------ pkg/build/gobuild_test.go | 4 +- pkg/build/options.go | 10 ++-- pkg/commands/options/build.go | 4 +- pkg/commands/resolver.go | 2 +- 5 files changed, 55 insertions(+), 55 deletions(-) diff --git a/pkg/build/gobuild.go b/pkg/build/gobuild.go index c971cb408f..960d006abe 100644 --- a/pkg/build/gobuild.go +++ b/pkg/build/gobuild.go @@ -70,20 +70,20 @@ type platformMatcher struct { } type gobuild struct { - ctx context.Context - getBase GetBase - creationTime v1.Time - kodataCreationTime v1.Time - build builder - sbom sbomber - disableOptimizations bool - trimpath bool - preserveDockerMediaType bool - buildConfigs map[string]Config - platformMatcher *platformMatcher - dir string - labels map[string]string - semaphore *semaphore.Weighted + ctx context.Context + getBase GetBase + creationTime v1.Time + kodataCreationTime v1.Time + build builder + sbom sbomber + disableOptimizations bool + trimpath bool + preserveMediaType bool + buildConfigs map[string]Config + platformMatcher *platformMatcher + dir string + labels map[string]string + semaphore *semaphore.Weighted cache *layerCache } @@ -92,20 +92,20 @@ type gobuild struct { type Option func(*gobuildOpener) error type gobuildOpener struct { - ctx context.Context - getBase GetBase - creationTime v1.Time - kodataCreationTime v1.Time - build builder - sbom sbomber - disableOptimizations bool - trimpath bool - preserveDockerMediaType bool - buildConfigs map[string]Config - platforms []string - labels map[string]string - dir string - jobs int + ctx context.Context + getBase GetBase + creationTime v1.Time + kodataCreationTime v1.Time + build builder + sbom sbomber + disableOptimizations bool + trimpath bool + preserveMediaType bool + buildConfigs map[string]Config + platforms []string + labels map[string]string + dir string + jobs int } func (gbo *gobuildOpener) Open() (Interface, error) { @@ -120,19 +120,19 @@ func (gbo *gobuildOpener) Open() (Interface, error) { gbo.jobs = runtime.GOMAXPROCS(0) } return &gobuild{ - ctx: gbo.ctx, - getBase: gbo.getBase, - creationTime: gbo.creationTime, - kodataCreationTime: gbo.kodataCreationTime, - build: gbo.build, - sbom: gbo.sbom, - disableOptimizations: gbo.disableOptimizations, - trimpath: gbo.trimpath, - preserveDockerMediaType: gbo.preserveDockerMediaType, - buildConfigs: gbo.buildConfigs, - labels: gbo.labels, - dir: gbo.dir, - platformMatcher: matcher, + ctx: gbo.ctx, + getBase: gbo.getBase, + creationTime: gbo.creationTime, + kodataCreationTime: gbo.kodataCreationTime, + build: gbo.build, + sbom: gbo.sbom, + disableOptimizations: gbo.disableOptimizations, + trimpath: gbo.trimpath, + preserveMediaType: gbo.preserveMediaType, + buildConfigs: gbo.buildConfigs, + labels: gbo.labels, + dir: gbo.dir, + platformMatcher: matcher, cache: &layerCache{ buildToDiff: map[string]buildIDToDiffID{}, diffToDesc: map[string]diffIDToDescriptor{}, @@ -698,7 +698,7 @@ func (g *gobuild) buildOne(ctx context.Context, refStr string, base v1.Image, pl ref := newRef(refStr) baseType := types.OCIManifestSchema1 - if g.preserveDockerMediaType { + if g.preserveMediaType { var err error baseType, err = base.MediaType() if err != nil { @@ -897,7 +897,7 @@ func (g *gobuild) Build(ctx context.Context, s string) (Result, error) { // Annotate the base image we pass to the build function with // annotations indicating the digest (and possibly tag) of the // base image. This will be inherited by the image produced. - if mt != types.DockerManifestList && !g.preserveDockerMediaType { + if mt != types.DockerManifestList && !g.preserveMediaType { anns := map[string]string{ specsv1.AnnotationBaseImageDigest: baseDigest.String(), } @@ -978,7 +978,7 @@ func (g *gobuild) buildAll(ctx context.Context, ref string, baseIndex v1.ImageIn } mt := types.OCIManifestSchema1 - if g.preserveDockerMediaType { + if g.preserveMediaType { mt = desc.MediaType } @@ -999,7 +999,7 @@ func (g *gobuild) buildAll(ctx context.Context, ref string, baseIndex v1.ImageIn } baseType := types.OCIImageIndex - if g.preserveDockerMediaType { + if g.preserveMediaType { baseType, err = baseIndex.MediaType() if err != nil { return nil, err diff --git a/pkg/build/gobuild_test.go b/pkg/build/gobuild_test.go index 0f3035fdd9..7c6449ff56 100644 --- a/pkg/build/gobuild_test.go +++ b/pkg/build/gobuild_test.go @@ -949,7 +949,7 @@ func TestGoBuildIndex(t *testing.T) { }) } -func TestPreserveDockerMediaType(t *testing.T) { +func TestPreserveMediaType(t *testing.T) { mustRandomImage := func(t *testing.T) v1.Image { img, err := random.Image(1, 1) if err != nil { @@ -1008,7 +1008,7 @@ func TestPreserveDockerMediaType(t *testing.T) { "", WithBaseImages(func(context.Context, string) (name.Reference, Result, error) { return baseRef, c.base, nil }), WithPlatforms("all"), - WithPreserveDockerMediaType(c.preserve), + WithPreserveMediaType(c.preserve), withBuilder(writeTempFile), ) if err != nil { diff --git a/pkg/build/options.go b/pkg/build/options.go index 3bd02e9940..ee25949bc3 100644 --- a/pkg/build/options.go +++ b/pkg/build/options.go @@ -73,12 +73,12 @@ func WithTrimpath(v bool) Option { } } -// WithPreserveDockerMediaType is a functional option that controls whether to -// preserve Docker media types from base images. If false, images that are -// produced will use OCI media types. -func WithPreserveDockerMediaType(v bool) Option { +// WithPreserveMediaType is a functional option that controls whether to +// preserve media types from base images. If false, images that are produced +// will use OCI media types instead. +func WithPreserveMediaType(v bool) Option { return func(gbo *gobuildOpener) error { - gbo.preserveDockerMediaType = v + gbo.preserveMediaType = v return nil } } diff --git a/pkg/commands/options/build.go b/pkg/commands/options/build.go index af9f07f61e..6db10667ef 100644 --- a/pkg/commands/options/build.go +++ b/pkg/commands/options/build.go @@ -69,7 +69,7 @@ type BuildOptions struct { BuildConfigs map[string]build.Config // If true, don't convert Docker-typed base images to OCI when building. - PreserveDockerMediaType bool + PreserveMediaType bool } func AddBuildOptions(cmd *cobra.Command, bo *BuildOptions) { @@ -84,7 +84,7 @@ func AddBuildOptions(cmd *cobra.Command, bo *BuildOptions) { cmd.Flags().StringSliceVar(&bo.Labels, "image-label", []string{}, "Which labels (key=value) to add to the image.") - cmd.Flags().BoolVar(&bo.PreserveDockerMediaType, "preserve-docker-media-type", false, "If false, push images in OCI format regardless of base image format") + cmd.Flags().BoolVar(&bo.PreserveMediaType, "preserve-media-type", false, "If false, push images in OCI format regardless of base image format") bo.Trimpath = true } diff --git a/pkg/commands/resolver.go b/pkg/commands/resolver.go index 97c4e14abe..9526b6402a 100644 --- a/pkg/commands/resolver.go +++ b/pkg/commands/resolver.go @@ -108,7 +108,7 @@ func gobuildOptions(bo *options.BuildOptions) ([]build.Option, error) { opts = append(opts, build.WithSPDX(version())) } opts = append(opts, build.WithTrimpath(bo.Trimpath)) - opts = append(opts, build.WithPreserveDockerMediaType(bo.PreserveDockerMediaType)) + opts = append(opts, build.WithPreserveMediaType(bo.PreserveMediaType)) for _, lf := range bo.Labels { parts := strings.SplitN(lf, "=", 2) if len(parts) != 2 { From c721fdb186f24ca02e7842900873ef2e59ca4ce7 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Fri, 4 Mar 2022 14:06:46 -0500 Subject: [PATCH 5/5] docs --- doc/ko_apply.md | 2 +- doc/ko_build.md | 36 ++++++++++++++++++------------------ doc/ko_create.md | 2 +- doc/ko_resolve.md | 42 +++++++++++++++++++++--------------------- doc/ko_run.md | 36 ++++++++++++++++++------------------ 5 files changed, 59 insertions(+), 59 deletions(-) diff --git a/doc/ko_apply.md b/doc/ko_apply.md index 6f12d2809c..73288128c2 100644 --- a/doc/ko_apply.md +++ b/doc/ko_apply.md @@ -69,8 +69,8 @@ ko apply -f FILENAME [flags] --oci-layout-path string Path to save the OCI image layout of the built images --password string Password for basic authentication to the API server (DEPRECATED) --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* - --preserve-docker-media-type If false, push images in OCI format regardless of base image format -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --preserve-media-type If false, push images in OCI format regardless of base image format --push Push images to KO_DOCKER_REPO (default true) -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (DEPRECATED) diff --git a/doc/ko_build.md b/doc/ko_build.md index 8e02bf502b..9ed0a9e364 100644 --- a/doc/ko_build.md +++ b/doc/ko_build.md @@ -43,24 +43,24 @@ ko build IMPORTPATH... [flags] ### Options ``` - --bare Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags). - -B, --base-import-paths Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags). - --disable-optimizations Disable optimizations when building Go code. Useful when you want to interactively debug the created container. - -h, --help help for build - --image-label strings Which labels (key=value) to add to the image. - --image-refs string Path to file where a list of the published image references will be written. - --insecure-registry Whether to skip TLS verification on the registry - -j, --jobs int The maximum number of concurrent builds (default GOMAXPROCS) - -L, --local Load into images to local docker daemon. - --oci-layout-path string Path to save the OCI image layout of the built images - --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* - --preserve-docker-media-type If false, push images in OCI format regardless of base image format - -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. - --push Push images to KO_DOCKER_REPO (default true) - --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") - --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. - -t, --tags strings Which tags to use for the produced image instead of the default 'latest' tag (may not work properly with --base-import-paths or --bare). (default [latest]) - --tarball string File to save images tarballs + --bare Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags). + -B, --base-import-paths Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags). + --disable-optimizations Disable optimizations when building Go code. Useful when you want to interactively debug the created container. + -h, --help help for build + --image-label strings Which labels (key=value) to add to the image. + --image-refs string Path to file where a list of the published image references will be written. + --insecure-registry Whether to skip TLS verification on the registry + -j, --jobs int The maximum number of concurrent builds (default GOMAXPROCS) + -L, --local Load into images to local docker daemon. + --oci-layout-path string Path to save the OCI image layout of the built images + --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* + -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --preserve-media-type If false, push images in OCI format regardless of base image format + --push Push images to KO_DOCKER_REPO (default true) + --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") + --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. + -t, --tags strings Which tags to use for the produced image instead of the default 'latest' tag (may not work properly with --base-import-paths or --bare). (default [latest]) + --tarball string File to save images tarballs ``` ### Options inherited from parent commands diff --git a/doc/ko_create.md b/doc/ko_create.md index f594aadf02..d6d6b2a529 100644 --- a/doc/ko_create.md +++ b/doc/ko_create.md @@ -69,8 +69,8 @@ ko create -f FILENAME [flags] --oci-layout-path string Path to save the OCI image layout of the built images --password string Password for basic authentication to the API server (DEPRECATED) --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* - --preserve-docker-media-type If false, push images in OCI format regardless of base image format -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --preserve-media-type If false, push images in OCI format regardless of base image format --push Push images to KO_DOCKER_REPO (default true) -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. --request-timeout string The length of time to wait before giving up on a single server request. Non-zero values should contain a corresponding time unit (e.g. 1s, 2m, 3h). A value of zero means don't timeout requests. (DEPRECATED) diff --git a/doc/ko_resolve.md b/doc/ko_resolve.md index 78e772694d..d046e59d89 100644 --- a/doc/ko_resolve.md +++ b/doc/ko_resolve.md @@ -38,27 +38,27 @@ ko resolve -f FILENAME [flags] ### Options ``` - --bare Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags). - -B, --base-import-paths Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags). - --disable-optimizations Disable optimizations when building Go code. Useful when you want to interactively debug the created container. - -f, --filename strings Filename, directory, or URL to files to use to create the resource - -h, --help help for resolve - --image-label strings Which labels (key=value) to add to the image. - --image-refs string Path to file where a list of the published image references will be written. - --insecure-registry Whether to skip TLS verification on the registry - -j, --jobs int The maximum number of concurrent builds (default GOMAXPROCS) - -L, --local Load into images to local docker daemon. - --oci-layout-path string Path to save the OCI image layout of the built images - --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* - --preserve-docker-media-type If false, push images in OCI format regardless of base image format - -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. - --push Push images to KO_DOCKER_REPO (default true) - -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. - --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") - -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2) - --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. - -t, --tags strings Which tags to use for the produced image instead of the default 'latest' tag (may not work properly with --base-import-paths or --bare). (default [latest]) - --tarball string File to save images tarballs + --bare Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags). + -B, --base-import-paths Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags). + --disable-optimizations Disable optimizations when building Go code. Useful when you want to interactively debug the created container. + -f, --filename strings Filename, directory, or URL to files to use to create the resource + -h, --help help for resolve + --image-label strings Which labels (key=value) to add to the image. + --image-refs string Path to file where a list of the published image references will be written. + --insecure-registry Whether to skip TLS verification on the registry + -j, --jobs int The maximum number of concurrent builds (default GOMAXPROCS) + -L, --local Load into images to local docker daemon. + --oci-layout-path string Path to save the OCI image layout of the built images + --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* + -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --preserve-media-type If false, push images in OCI format regardless of base image format + --push Push images to KO_DOCKER_REPO (default true) + -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. + --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") + -l, --selector string Selector (label query) to filter on, supports '=', '==', and '!='.(e.g. -l key1=value1,key2=value2) + --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. + -t, --tags strings Which tags to use for the produced image instead of the default 'latest' tag (may not work properly with --base-import-paths or --bare). (default [latest]) + --tarball string File to save images tarballs ``` ### Options inherited from parent commands diff --git a/doc/ko_run.md b/doc/ko_run.md index ebecc7948d..f7a2c8aa30 100644 --- a/doc/ko_run.md +++ b/doc/ko_run.md @@ -30,24 +30,24 @@ ko run IMPORTPATH [flags] ### Options ``` - --bare Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags). - -B, --base-import-paths Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags). - --disable-optimizations Disable optimizations when building Go code. Useful when you want to interactively debug the created container. - -h, --help help for run - --image-label strings Which labels (key=value) to add to the image. - --image-refs string Path to file where a list of the published image references will be written. - --insecure-registry Whether to skip TLS verification on the registry - -j, --jobs int The maximum number of concurrent builds (default GOMAXPROCS) - -L, --local Load into images to local docker daemon. - --oci-layout-path string Path to save the OCI image layout of the built images - --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* - --preserve-docker-media-type If false, push images in OCI format regardless of base image format - -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. - --push Push images to KO_DOCKER_REPO (default true) - --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") - --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. - -t, --tags strings Which tags to use for the produced image instead of the default 'latest' tag (may not work properly with --base-import-paths or --bare). (default [latest]) - --tarball string File to save images tarballs + --bare Whether to just use KO_DOCKER_REPO without additional context (may not work properly with --tags). + -B, --base-import-paths Whether to use the base path without MD5 hash after KO_DOCKER_REPO (may not work properly with --tags). + --disable-optimizations Disable optimizations when building Go code. Useful when you want to interactively debug the created container. + -h, --help help for run + --image-label strings Which labels (key=value) to add to the image. + --image-refs string Path to file where a list of the published image references will be written. + --insecure-registry Whether to skip TLS verification on the registry + -j, --jobs int The maximum number of concurrent builds (default GOMAXPROCS) + -L, --local Load into images to local docker daemon. + --oci-layout-path string Path to save the OCI image layout of the built images + --platform strings Which platform to use when pulling a multi-platform base. Format: all | [/[/]][,platform]* + -P, --preserve-import-paths Whether to preserve the full import path after KO_DOCKER_REPO. + --preserve-media-type If false, push images in OCI format regardless of base image format + --push Push images to KO_DOCKER_REPO (default true) + --sbom string The SBOM media type to use (none will disable SBOM synthesis and upload, also supports: spdx, go.version-m). (default "spdx") + --tag-only Include tags but not digests in resolved image references. Useful when digests are not preserved when images are repopulated. + -t, --tags strings Which tags to use for the produced image instead of the default 'latest' tag (may not work properly with --base-import-paths or --bare). (default [latest]) + --tarball string File to save images tarballs ``` ### Options inherited from parent commands