From acfceca6e0b999f071609941311fdeb826a0e0f5 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 5 Jun 2024 20:15:09 +0000 Subject: [PATCH 1/7] fix --- examples/podinfo-flux/zarf.yaml | 28 +--------------------------- src/internal/packager/images/pull.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/examples/podinfo-flux/zarf.yaml b/examples/podinfo-flux/zarf.yaml index b06283e46e..3f90c0f916 100644 --- a/examples/podinfo-flux/zarf.yaml +++ b/examples/podinfo-flux/zarf.yaml @@ -13,33 +13,7 @@ components: files: - flux-install.yaml images: - - ghcr.io/fluxcd/kustomize-controller:v0.27.1 - - ghcr.io/fluxcd/source-controller:v0.28.0 - - - name: podinfo-via-flux - description: Example deployment via flux using the famous podinfo example - required: true - manifests: - - name: podinfo-via-flux - namespace: podinfo - files: - - podinfo-source.yaml - - podinfo-kustomization.yaml - repos: - - https://github.com/stefanprodan/podinfo.git - images: - - ghcr.io/stefanprodan/podinfo:6.3.3 - actions: - onDeploy: - after: - # This will use a wait action to wait for the podinfo pod to be ready - - description: Podinfo pods to be ready via wait action - wait: - cluster: - kind: pod - name: app=podinfo - namespace: podinfo - condition: ready + - alpine:austin # YAML keys starting with `x-` are custom keys that are ignored by the Zarf CLI # The `x-mdx` key is used to render the markdown content for https://docs.zarf.dev/ref/examples diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index c440f8d33d..3b0e05dddc 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -246,6 +246,34 @@ func Pull(ctx context.Context, cfg PullConfig) (map[transform.Image]v1.Image, er doneSaving <- nil <-doneSaving + // This fixes an issue on amd64, when pulling images from the local docker daemon, + // while using the docker containerd runtime. Crane incorrectly names the blob of the docker image config + // to a sha that does not match the contents + // https://github.com/defenseunicorns/zarf/issues/2584 + // This is a band aid fix while we wait for crane and the docker to create the permanent fix + blobDirectory := filepath.Join(cfg.DestinationDirectory, "images", "blobs", "sha256") + err = filepath.Walk(blobDirectory, func(path string, _ os.FileInfo, err error) error { + if err != nil { + return err + } + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + hash, err := helpers.GetSHA256Hash(file) + if err != nil { + return err + } + + return os.Rename(path, hash) + }) + if err != nil { + return nil, err + } + return fetched, nil } From 8475a9eb4f93f7014b8e4de17897bd40812f74a5 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 5 Jun 2024 21:09:38 +0000 Subject: [PATCH 2/7] WIP that doesn't really work --- src/internal/packager/images/pull.go | 28 ---------------------------- src/pkg/layout/image.go | 17 ++++++++++++++--- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 3b0e05dddc..c440f8d33d 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -246,34 +246,6 @@ func Pull(ctx context.Context, cfg PullConfig) (map[transform.Image]v1.Image, er doneSaving <- nil <-doneSaving - // This fixes an issue on amd64, when pulling images from the local docker daemon, - // while using the docker containerd runtime. Crane incorrectly names the blob of the docker image config - // to a sha that does not match the contents - // https://github.com/defenseunicorns/zarf/issues/2584 - // This is a band aid fix while we wait for crane and the docker to create the permanent fix - blobDirectory := filepath.Join(cfg.DestinationDirectory, "images", "blobs", "sha256") - err = filepath.Walk(blobDirectory, func(path string, _ os.FileInfo, err error) error { - if err != nil { - return err - } - - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - hash, err := helpers.GetSHA256Hash(file) - if err != nil { - return err - } - - return os.Rename(path, hash) - }) - if err != nil { - return nil, err - } - return fetched, nil } diff --git a/src/pkg/layout/image.go b/src/pkg/layout/image.go index 15348bf1d3..dc19789034 100644 --- a/src/pkg/layout/image.go +++ b/src/pkg/layout/image.go @@ -5,10 +5,12 @@ package layout import ( + "os" "path/filepath" "slices" + "github.com/defenseunicorns/pkg/helpers" v1 "github.com/google/go-containerregistry/pkg/v1" ) @@ -25,9 +27,18 @@ func (i *Images) AddBlob(blob string) { if len(blob) != 64 { return } - abs := filepath.Join(i.Base, "blobs", "sha256", blob) - if !slices.Contains(i.Blobs, abs) { - i.Blobs = append(i.Blobs, abs) + layerPath := filepath.Join(i.Base, "blobs", "sha256") + abs := filepath.Join(layerPath, blob) + absSha, err := helpers.GetSHA256OfFile(abs) + if err != nil { + return + } + newPath := filepath.Join(layerPath, absSha) + if absSha != blob { + os.Rename(abs, newPath) + } + if !slices.Contains(i.Blobs, newPath) { + i.Blobs = append(i.Blobs, newPath) } } From 7483a0978d0603ad9744d7454fe7b4f5889db110 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 6 Jun 2024 17:24:07 +0000 Subject: [PATCH 3/7] changing how we add blobs --- src/internal/packager/images/pull.go | 26 ++++++++++++++++++++++++++ src/pkg/layout/image.go | 24 ++++++++---------------- 2 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index c440f8d33d..6eb63398b7 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -246,6 +246,32 @@ func Pull(ctx context.Context, cfg PullConfig) (map[transform.Image]v1.Image, er doneSaving <- nil <-doneSaving + // Needed because when pulling form the local docker daemon, while using the docker containerd runtime + // Crane incorrectly names the blob of the docker image config to a sha that does not match the contents + // https://github.com/defenseunicorns/zarf/issues/2584 + // This is a band aid fix while we wait for crane and or docker to create the permanent fix + blobDir := filepath.Join(cfg.DestinationDirectory, "blobs", "sha256") + err = filepath.Walk(blobDir, func(path string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + if fi.IsDir() { + return nil + } + + hash, err := helpers.GetSHA256OfFile(path) + if err != nil { + return err + } + newFile := filepath.Join(blobDir, hash) + + return os.Rename(path, newFile) + }) + if err != nil { + return nil, err + } + return fetched, nil } diff --git a/src/pkg/layout/image.go b/src/pkg/layout/image.go index dc19789034..00358a50ee 100644 --- a/src/pkg/layout/image.go +++ b/src/pkg/layout/image.go @@ -5,12 +5,10 @@ package layout import ( - "os" "path/filepath" "slices" - "github.com/defenseunicorns/pkg/helpers" v1 "github.com/google/go-containerregistry/pkg/v1" ) @@ -27,18 +25,9 @@ func (i *Images) AddBlob(blob string) { if len(blob) != 64 { return } - layerPath := filepath.Join(i.Base, "blobs", "sha256") - abs := filepath.Join(layerPath, blob) - absSha, err := helpers.GetSHA256OfFile(abs) - if err != nil { - return - } - newPath := filepath.Join(layerPath, absSha) - if absSha != blob { - os.Rename(abs, newPath) - } - if !slices.Contains(i.Blobs, newPath) { - i.Blobs = append(i.Blobs, newPath) + abs := filepath.Join(i.Base, "blobs", "sha256", blob) + if !slices.Contains(i.Blobs, abs) { + i.Blobs = append(i.Blobs, abs) } } @@ -55,11 +44,14 @@ func (i *Images) AddV1Image(img v1.Image) error { } i.AddBlob(digest.Hex) } - imgCfgSha, err := img.ConfigName() + + manifest, err := img.Manifest() if err != nil { return err } - i.AddBlob(imgCfgSha.Hex) + // Cannot use img.ConfigName to get this value because of an upstream bug in crane / docker using the containerd runtime + // https://github.com/defenseunicorns/zarf/issues/2584 + i.AddBlob(manifest.Config.Digest.Hex) manifestSha, err := img.Digest() if err != nil { return err From ba965f9482e8c1c4592c721ecad7423e3f6f05fd Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 6 Jun 2024 17:32:52 +0000 Subject: [PATCH 4/7] revert zarf.yaml --- examples/podinfo-flux/zarf.yaml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/examples/podinfo-flux/zarf.yaml b/examples/podinfo-flux/zarf.yaml index 3f90c0f916..b06283e46e 100644 --- a/examples/podinfo-flux/zarf.yaml +++ b/examples/podinfo-flux/zarf.yaml @@ -13,7 +13,33 @@ components: files: - flux-install.yaml images: - - alpine:austin + - ghcr.io/fluxcd/kustomize-controller:v0.27.1 + - ghcr.io/fluxcd/source-controller:v0.28.0 + + - name: podinfo-via-flux + description: Example deployment via flux using the famous podinfo example + required: true + manifests: + - name: podinfo-via-flux + namespace: podinfo + files: + - podinfo-source.yaml + - podinfo-kustomization.yaml + repos: + - https://github.com/stefanprodan/podinfo.git + images: + - ghcr.io/stefanprodan/podinfo:6.3.3 + actions: + onDeploy: + after: + # This will use a wait action to wait for the podinfo pod to be ready + - description: Podinfo pods to be ready via wait action + wait: + cluster: + kind: pod + name: app=podinfo + namespace: podinfo + condition: ready # YAML keys starting with `x-` are custom keys that are ignored by the Zarf CLI # The `x-mdx` key is used to render the markdown content for https://docs.zarf.dev/ref/examples From 353b7d0dd156ef62a2f83f63a382e71470d2e22c Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 6 Jun 2024 17:49:38 +0000 Subject: [PATCH 5/7] removing whitespacer for phillip --- src/internal/packager/images/pull.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 6eb63398b7..70cd1d3dc1 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -265,7 +265,6 @@ func Pull(ctx context.Context, cfg PullConfig) (map[transform.Image]v1.Image, er return err } newFile := filepath.Join(blobDir, hash) - return os.Rename(path, newFile) }) if err != nil { From eb4ff6f3eccbf2469c817a9adef318ecaa88c2e9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 6 Jun 2024 17:50:32 +0000 Subject: [PATCH 6/7] removing whitespace --- src/pkg/layout/image.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pkg/layout/image.go b/src/pkg/layout/image.go index 00358a50ee..7d236410bf 100644 --- a/src/pkg/layout/image.go +++ b/src/pkg/layout/image.go @@ -44,7 +44,6 @@ func (i *Images) AddV1Image(img v1.Image) error { } i.AddBlob(digest.Hex) } - manifest, err := img.Manifest() if err != nil { return err From cacc9814cc144caf7afd5d920b1d400400218909 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 6 Jun 2024 20:35:48 +0000 Subject: [PATCH 7/7] spelling --- src/internal/packager/images/pull.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 70cd1d3dc1..9063075879 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -246,7 +246,7 @@ func Pull(ctx context.Context, cfg PullConfig) (map[transform.Image]v1.Image, er doneSaving <- nil <-doneSaving - // Needed because when pulling form the local docker daemon, while using the docker containerd runtime + // Needed because when pulling from the local docker daemon, while using the docker containerd runtime // Crane incorrectly names the blob of the docker image config to a sha that does not match the contents // https://github.com/defenseunicorns/zarf/issues/2584 // This is a band aid fix while we wait for crane and or docker to create the permanent fix