From 7d1988dd2455c5d6a757857c3723c1e799e904d4 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 9 Apr 2024 16:30:41 +0000 Subject: [PATCH 01/38] core of what I'm looking for working --- src/config/config.go | 8 +++--- src/config/lang/english.go | 19 ++++++++------- src/internal/packager/images/pull.go | 19 ++++++++++++++- src/pkg/packager/creator/normal.go | 4 +++ src/test/e2e/14_create_sha_index_test.go | 21 ++++++++++++++++ src/test/packages/14-index-sha/zarf.yaml | 31 ++++++++++++++++++++++++ 6 files changed, 88 insertions(+), 14 deletions(-) create mode 100644 src/test/e2e/14_create_sha_index_test.go create mode 100644 src/test/packages/14-index-sha/zarf.yaml diff --git a/src/config/config.go b/src/config/config.go index b8e3e2d386..c8188f421e 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -134,12 +134,12 @@ func GetCraneOptions(insecure bool, archs ...string) []crane.Option { options = append(options, crane.Insecure, crane.WithTransport(roundTripper)) } + if archs != nil { + options = append(options, crane.WithPlatform(&v1.Platform{OS: "linux", Architecture: GetArch(archs...)})) + } + // Add the image platform info options = append(options, - crane.WithPlatform(&v1.Platform{ - OS: "linux", - Architecture: GetArch(archs...), - }), crane.WithUserAgent("zarf"), crane.WithNoClobber(true), // TODO: (@WSTARR) this is set to limit pushes to registry pods and reduce the likelihood that crane will get stuck. diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 9c9219a231..52a08864f3 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -477,7 +477,7 @@ $ zarf tools registry digest reg.example.com/stefanprodan/podinfo:6.4.0 CmdToolsGetGitPasswdShort = "[Deprecated] Returns the push user's password for the Git server" CmdToolsGetGitPasswdLong = "[Deprecated] Reads the password for a user with push access to the configured Git server in Zarf State. Note that this command has been replaced by 'zarf tools get-creds git' and will be removed in Zarf v1.0.0." CmdToolsGetGitPasswdDeprecation = "Deprecated: This command has been replaced by 'zarf tools get-creds git' and will be removed in Zarf v1.0.0." - CmdToolsYqExample = ` + CmdToolsYqExample = ` # yq defaults to 'eval' command if no command is specified. See "zarf tools yq eval --help" for more examples. # read the "stuff" node from "myfile.yml" @@ -505,10 +505,10 @@ cat file2.yml | zarf tools yq ea '.a.b' file1.yml - file3.yml ` CmdToolsYqEvalExample = ` # Reads field under the given path for each file -zarf tools yq e '.a.b' f1.yml f2.yml +zarf tools yq e '.a.b' f1.yml f2.yml # Prints out the file -zarf tools yq e sample.yaml +zarf tools yq e sample.yaml # Pipe from STDIN ## use '-' as a filename to pipe from STDIN @@ -516,10 +516,10 @@ cat file2.yml | zarf tools yq e '.a.b' file1.yml - file3.yml # Creates a new yaml document ## Note that editing an empty file does not work. -zarf tools yq e -n '.a.b.c = "cat"' +zarf tools yq e -n '.a.b.c = "cat"' # Update a file inplace -zarf tools yq e '.a.b = "cool"' -i file.yaml +zarf tools yq e '.a.b = "cool"' -i file.yaml ` CmdToolsMonitorShort = "Launches a terminal UI to monitor the connected cluster using K9s." @@ -727,10 +727,11 @@ const ( // Collection of reusable error messages. var ( - ErrInitNotFound = errors.New("this command requires a zarf-init package, but one was not found on the local system. Re-run the last command again without '--confirm' to download the package") - ErrUnableToCheckArch = errors.New("unable to get the configured cluster's architecture") - ErrInterrupt = errors.New("execution cancelled due to an interrupt") - ErrUnableToGetPackages = errors.New("unable to load the Zarf Package data from the cluster") + ErrInitNotFound = errors.New("this command requires a zarf-init package, but one was not found on the local system. Re-run the last command again without '--confirm' to download the package") + ErrUnableToCheckArch = errors.New("unable to get the configured cluster's architecture") + ErrInterrupt = errors.New("execution cancelled due to an interrupt") + ErrUnableToGetPackages = errors.New("unable to load the Zarf Package data from the cluster") + ErrUnsupportedImageType = errors.New("zarf does not currently support image indexes or docker manifest lists") ) // Collection of reusable warn messages. diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index df839e4f07..24557b411b 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -6,12 +6,14 @@ package images import ( "context" + "encoding/json" "fmt" "path/filepath" "strings" "github.com/defenseunicorns/pkg/helpers" "github.com/defenseunicorns/zarf/src/config" + "github.com/defenseunicorns/zarf/src/config/lang" "github.com/defenseunicorns/zarf/src/pkg/layout" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/transform" @@ -25,6 +27,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/empty" clayout "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/partial" + ctypes "github.com/google/go-containerregistry/pkg/v1/types" "github.com/moby/moby/client" ) @@ -271,7 +274,7 @@ func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Im if err != nil { return nil, false, err } - } else if _, err := crane.Manifest(src, config.GetCraneOptions(i.Insecure, i.Architectures...)...); err != nil { + } else if manifest, err := crane.Manifest(src, config.GetCraneOptions(i.Insecure)...); err != nil { // If crane is unable to pull the image, try to load it from the local docker daemon. message.Notef("Falling back to local 'docker' images, failed to find the manifest on a remote: %s", err.Error()) @@ -308,6 +311,20 @@ func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Im return nil, false, fmt.Errorf("failed to load image from docker daemon: %w", err) } } else { + var idx v1.IndexManifest + if err := json.Unmarshal(manifest, &idx); err != nil { + return nil, false, lang.ErrUnsupportedImageType + } + + if strings.Contains(src, "@") && (idx.MediaType == ctypes.OCIImageIndex || idx.MediaType == ctypes.DockerManifestList) { + imageOptions := "please select one of the images below based on your platform" + imageBaseName := strings.Split(src, "@")[0] + for _, manifest := range idx.Manifests { + imageOptions = fmt.Sprintf("%s\n %s@%s for platform %v", imageOptions, imageBaseName, manifest.Digest, manifest.Platform) + } + return nil, false, fmt.Errorf("%w: %s", lang.ErrUnsupportedImageType, imageOptions) + } + // Manifest was found, so use crane to pull the image. if img, err = crane.Pull(src, config.GetCraneOptions(i.Insecure, i.Architectures...)...); err != nil { return nil, false, fmt.Errorf("failed to pull image: %w", err) diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 5693d32d98..75a3cf4a94 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -186,6 +186,10 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, components []types. } pulled, err = imgConfig.PullAll() + if errors.Is(err, lang.ErrUnsupportedImageType) { + message.Fatal(err, err.Error()) + } + return err } diff --git a/src/test/e2e/14_create_sha_index_test.go b/src/test/e2e/14_create_sha_index_test.go new file mode 100644 index 0000000000..c766dea8dd --- /dev/null +++ b/src/test/e2e/14_create_sha_index_test.go @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: 2021-Present The Zarf Authors + +// Package test provides e2e tests for Zarf. +package test + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCreateIndexSha(t *testing.T) { + t.Log("E2E: Create Templating") + + _, stderr, err := e2e.Zarf("package", "create", "src/test/packages/14-index-sha", "--confirm") + // Not sure why this isn't working + require.Error(t, err) + require.Contains(t, stderr, "docker.io/defenseunicorns/zarf-game:multi-tile-dark@sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff") + +} diff --git a/src/test/packages/14-index-sha/zarf.yaml b/src/test/packages/14-index-sha/zarf.yaml new file mode 100644 index 0000000000..55c4d5aedc --- /dev/null +++ b/src/test/packages/14-index-sha/zarf.yaml @@ -0,0 +1,31 @@ +kind: ZarfPackageConfig +metadata: + name: index-sha + +components: + - name: baseline + required: true + images: + - docker.io/defenseunicorns/zarf-game:multi-tile-dark@sha256:0b694ca1c33afae97b7471488e07968599f1d2470c629f76af67145ca64428af + # - ghcr.io/defenseunicorns/zarf/agent:v0.32.6@sha256:05a82656df5466ce17c3e364c16792ae21ce68438bfe06eeab309d0520c16b48 +# +# Initial two tests to get working +# On create I can pull every image in and write to the zarf.yaml so that there is record of them +# On deploy I can both deploy every image to the internal registry and deploy + +# Name: docker.io/defenseunicorns/zarf-game:multi-tile-dark +# MediaType: application/vnd.docker.distribution.manifest.list.v2+json +# Digest: sha256:0b694ca1c33afae97b7471488e07968599f1d2470c629f76af67145ca64428af + +# Manifests: +# Name: docker.io/defenseunicorns/zarf-game:multi-tile-dark@sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff +# MediaType: application/vnd.docker.distribution.manifest.v2+json +# Platform: linux/arm/v7 + +# Name: docker.io/defenseunicorns/zarf-game:multi-tile-dark@sha256:e4d27fe4b7bf6d5cb7ef02ed0d33ec0846796c09d6ed4bd94c8b946119a01b09 +# MediaType: application/vnd.docker.distribution.manifest.v2+json +# Platform: linux/arm64 + +# Name: docker.io/defenseunicorns/zarf-game:multi-tile-dark@sha256:e81b1467b812019f8e8e81450728b084471806dc7e959b7beb9f39933c337e7d +# MediaType: application/vnd.docker.distribution.manifest.v2+json +# Platform: linux/amd6 From c7eae7791a306b54462568f1193d90f75dd1e137 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 9 Apr 2024 17:04:13 +0000 Subject: [PATCH 02/38] simplify --- src/test/e2e/14_create_sha_index_test.go | 13 +++++--- .../14-index-sha/image-index/zarf.yaml | 9 ++++++ .../14-index-sha/manifest-list/zarf.yaml | 9 ++++++ src/test/packages/14-index-sha/zarf.yaml | 31 ------------------- 4 files changed, 26 insertions(+), 36 deletions(-) create mode 100644 src/test/packages/14-index-sha/image-index/zarf.yaml create mode 100644 src/test/packages/14-index-sha/manifest-list/zarf.yaml delete mode 100644 src/test/packages/14-index-sha/zarf.yaml diff --git a/src/test/e2e/14_create_sha_index_test.go b/src/test/e2e/14_create_sha_index_test.go index c766dea8dd..4b564fc00e 100644 --- a/src/test/e2e/14_create_sha_index_test.go +++ b/src/test/e2e/14_create_sha_index_test.go @@ -10,12 +10,15 @@ import ( "github.com/stretchr/testify/require" ) -func TestCreateIndexSha(t *testing.T) { - t.Log("E2E: Create Templating") +func TestCreateIndexShaErrors(t *testing.T) { + t.Log("E2E: CreateIndexShaErrors") - _, stderr, err := e2e.Zarf("package", "create", "src/test/packages/14-index-sha", "--confirm") - // Not sure why this isn't working + _, stderr, err := e2e.Zarf("package", "create", "src/test/packages/14-index-sha/image-index", "--confirm") require.Error(t, err) - require.Contains(t, stderr, "docker.io/defenseunicorns/zarf-game:multi-tile-dark@sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff") + require.Contains(t, stderr, "ghcr.io/defenseunicorns/zarf/agent:v0.32.6@sha256:b3fabdc7d4ecd0f396016ef78da19002c39e3ace352ea0ae4baa2ce9d5958376") + + _, stderr, err = e2e.Zarf("package", "create", "src/test/packages/14-index-sha/manifest-list", "--confirm") + require.Error(t, err) + require.Contains(t, stderr, "docker.io/defenseunicorns/zarf-game@sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff") } diff --git a/src/test/packages/14-index-sha/image-index/zarf.yaml b/src/test/packages/14-index-sha/image-index/zarf.yaml new file mode 100644 index 0000000000..a29c4380b9 --- /dev/null +++ b/src/test/packages/14-index-sha/image-index/zarf.yaml @@ -0,0 +1,9 @@ +kind: ZarfPackageConfig +metadata: + name: image-index + +components: + - name: baseline + required: true + images: + - ghcr.io/defenseunicorns/zarf/agent:v0.32.6@sha256:05a82656df5466ce17c3e364c16792ae21ce68438bfe06eeab309d0520c16b48 diff --git a/src/test/packages/14-index-sha/manifest-list/zarf.yaml b/src/test/packages/14-index-sha/manifest-list/zarf.yaml new file mode 100644 index 0000000000..9d7ad76b20 --- /dev/null +++ b/src/test/packages/14-index-sha/manifest-list/zarf.yaml @@ -0,0 +1,9 @@ +kind: ZarfPackageConfig +metadata: + name: manifest-list + +components: + - name: baseline + required: true + images: + - defenseunicorns/zarf-game@sha256:0b694ca1c33afae97b7471488e07968599f1d2470c629f76af67145ca64428af diff --git a/src/test/packages/14-index-sha/zarf.yaml b/src/test/packages/14-index-sha/zarf.yaml deleted file mode 100644 index 55c4d5aedc..0000000000 --- a/src/test/packages/14-index-sha/zarf.yaml +++ /dev/null @@ -1,31 +0,0 @@ -kind: ZarfPackageConfig -metadata: - name: index-sha - -components: - - name: baseline - required: true - images: - - docker.io/defenseunicorns/zarf-game:multi-tile-dark@sha256:0b694ca1c33afae97b7471488e07968599f1d2470c629f76af67145ca64428af - # - ghcr.io/defenseunicorns/zarf/agent:v0.32.6@sha256:05a82656df5466ce17c3e364c16792ae21ce68438bfe06eeab309d0520c16b48 -# -# Initial two tests to get working -# On create I can pull every image in and write to the zarf.yaml so that there is record of them -# On deploy I can both deploy every image to the internal registry and deploy - -# Name: docker.io/defenseunicorns/zarf-game:multi-tile-dark -# MediaType: application/vnd.docker.distribution.manifest.list.v2+json -# Digest: sha256:0b694ca1c33afae97b7471488e07968599f1d2470c629f76af67145ca64428af - -# Manifests: -# Name: docker.io/defenseunicorns/zarf-game:multi-tile-dark@sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff -# MediaType: application/vnd.docker.distribution.manifest.v2+json -# Platform: linux/arm/v7 - -# Name: docker.io/defenseunicorns/zarf-game:multi-tile-dark@sha256:e4d27fe4b7bf6d5cb7ef02ed0d33ec0846796c09d6ed4bd94c8b946119a01b09 -# MediaType: application/vnd.docker.distribution.manifest.v2+json -# Platform: linux/arm64 - -# Name: docker.io/defenseunicorns/zarf-game:multi-tile-dark@sha256:e81b1467b812019f8e8e81450728b084471806dc7e959b7beb9f39933c337e7d -# MediaType: application/vnd.docker.distribution.manifest.v2+json -# Platform: linux/amd6 From 5dfe4dc42ca219852e5b103eb11baa8bd6e9f865 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 9 Apr 2024 17:05:44 +0000 Subject: [PATCH 03/38] whitespace --- .../100-cli-commands/zarf_tools_yq_eval.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md index 234ca9957e..3efe778a95 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md @@ -5,11 +5,11 @@ ## Synopsis -yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) +yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples. # Evaluate Sequence ## -This command iterates over each yaml document from each given file, applies the +This command iterates over each yaml document from each given file, applies the expression and prints the result in sequence. ``` @@ -21,10 +21,10 @@ zarf tools yq eval [expression] [yaml_file1]... [flags] ``` # Reads field under the given path for each file -zarf tools yq e '.a.b' f1.yml f2.yml +zarf tools yq e '.a.b' f1.yml f2.yml # Prints out the file -zarf tools yq e sample.yaml +zarf tools yq e sample.yaml # Pipe from STDIN # use '-' as a filename to pipe from STDIN @@ -32,10 +32,10 @@ cat file2.yml | zarf tools yq e '.a.b' file1.yml - file3.yml # Creates a new yaml document # Note that editing an empty file does not work. -zarf tools yq e -n '.a.b.c = "cat"' +zarf tools yq e -n '.a.b.c = "cat"' # Update a file inplace -zarf tools yq e '.a.b = "cool"' -i file.yaml +zarf tools yq e '.a.b = "cool"' -i file.yaml ``` From 48ab849a8d2d9ee8881fec0b89ab69d0467b57db Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 9 Apr 2024 17:10:07 +0000 Subject: [PATCH 04/38] platform has string() already --- src/internal/packager/images/pull.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 24557b411b..30d4d7820b 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -320,7 +320,7 @@ func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Im imageOptions := "please select one of the images below based on your platform" imageBaseName := strings.Split(src, "@")[0] for _, manifest := range idx.Manifests { - imageOptions = fmt.Sprintf("%s\n %s@%s for platform %v", imageOptions, imageBaseName, manifest.Digest, manifest.Platform) + imageOptions = fmt.Sprintf("%s\n %s@%s for platform %s", imageOptions, imageBaseName, manifest.Digest, manifest.Platform) } return nil, false, fmt.Errorf("%w: %s", lang.ErrUnsupportedImageType, imageOptions) } From 98c1a078fca159275c97d4cabdc62bdb3afe47d1 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 9 Apr 2024 17:21:39 +0000 Subject: [PATCH 05/38] whitespace --- docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md | 6 +++--- src/config/lang/english.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md index 3efe778a95..b7ea71a66e 100644 --- a/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md +++ b/docs/2-the-zarf-cli/100-cli-commands/zarf_tools_yq_eval.md @@ -5,11 +5,11 @@ ## Synopsis -yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) +yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples. # Evaluate Sequence ## -This command iterates over each yaml document from each given file, applies the +This command iterates over each yaml document from each given file, applies the expression and prints the result in sequence. ``` @@ -34,7 +34,7 @@ cat file2.yml | zarf tools yq e '.a.b' file1.yml - file3.yml # Note that editing an empty file does not work. zarf tools yq e -n '.a.b.c = "cat"' -# Update a file inplace +# Update a file in place zarf tools yq e '.a.b = "cool"' -i file.yaml ``` diff --git a/src/config/lang/english.go b/src/config/lang/english.go index 2c1db60341..2d008053ca 100644 --- a/src/config/lang/english.go +++ b/src/config/lang/english.go @@ -519,7 +519,7 @@ cat file2.yml | zarf tools yq e '.a.b' file1.yml - file3.yml ## Note that editing an empty file does not work. zarf tools yq e -n '.a.b.c = "cat"' -# Update a file inplace +# Update a file in place zarf tools yq e '.a.b = "cool"' -i file.yaml ` CmdToolsMonitorShort = "Launches a terminal UI to monitor the connected cluster using K9s." From f8aaf328b8ab68a300869a5ed64d28b55d2e3d5f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 9 Apr 2024 17:35:30 +0000 Subject: [PATCH 06/38] remove unneeded comment --- src/config/config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/config/config.go b/src/config/config.go index c8188f421e..63396255b0 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -138,7 +138,6 @@ func GetCraneOptions(insecure bool, archs ...string) []crane.Option { options = append(options, crane.WithPlatform(&v1.Platform{OS: "linux", Architecture: GetArch(archs...)})) } - // Add the image platform info options = append(options, crane.WithUserAgent("zarf"), crane.WithNoClobber(true), From 074b56c1318926b449b715a73a1a34ef14145c16 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 11 Apr 2024 11:49:58 -0400 Subject: [PATCH 07/38] retry with cancel implemented --- go.mod | 2 +- go.sum | 2 ++ src/pkg/packager/creator/normal.go | 7 ++++--- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index e065fd561d..22f660f703 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/anchore/clio v0.0.0-20240307182142-fb5fc4c9db3c github.com/anchore/stereoscope v0.0.1 github.com/anchore/syft v0.100.0 - github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411133953-c27224059905 + github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411154501-d8e8145af112 github.com/defenseunicorns/pkg/oci v0.0.1 github.com/derailed/k9s v0.31.7 github.com/distribution/reference v0.5.0 diff --git a/go.sum b/go.sum index cc40130846..d92cd77f9d 100644 --- a/go.sum +++ b/go.sum @@ -597,6 +597,8 @@ github.com/defenseunicorns/pkg/helpers v1.0.0 h1:0o3Rs+J/g0UemZHcENBS1Z2Qw2y4FIU github.com/defenseunicorns/pkg/helpers v1.0.0/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411133953-c27224059905 h1:x4KxchKd18SoYFAb/UmqpsJVhOVMB1VYJsVxt7w3s9s= github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411133953-c27224059905/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= +github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411154501-d8e8145af112 h1:Z6MQCjpKBA1CQfr3XqaRQsjXt+LYtNzfT4Adm53mfyE= +github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411154501-d8e8145af112/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= github.com/defenseunicorns/pkg/oci v0.0.1 h1:EFRp3NeiwzhOWKpQ6mAxi0l9chnrAvDcIgjMr0o0fkM= github.com/defenseunicorns/pkg/oci v0.0.1/go.mod h1:zVBgRjckEAhfdvbnQrnfOP/3M/GYJkIgWtJtY7pjYdo= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M= diff --git a/src/pkg/packager/creator/normal.go b/src/pkg/packager/creator/normal.go index 75a3cf4a94..20b239e8f5 100644 --- a/src/pkg/packager/creator/normal.go +++ b/src/pkg/packager/creator/normal.go @@ -176,6 +176,7 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, components []types. var pulled []images.ImgInfo var err error + ctx, cancel := context.WithCancel(context.TODO()) doPull := func() error { imgConfig := images.ImageConfig{ ImagesPath: dst.Images.Base, @@ -187,14 +188,14 @@ func (pc *PackageCreator) Assemble(dst *layout.PackagePaths, components []types. pulled, err = imgConfig.PullAll() if errors.Is(err, lang.ErrUnsupportedImageType) { - message.Fatal(err, err.Error()) + cancel() } return err } - if err := helpers.Retry(doPull, 3, 5*time.Second, message.Warnf); err != nil { - return fmt.Errorf("unable to pull images after 3 attempts: %w", err) + if err := helpers.RetryWithContext(ctx, doPull, 3, 5*time.Second, message.Warnf); err != nil { + return fmt.Errorf("unable to pull images: %w", err) } for _, imgInfo := range pulled { From 4c862c946d19b16c049c13c819052edfce58da0b Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 16 Apr 2024 13:27:58 +0000 Subject: [PATCH 08/38] changed error message --- go.mod | 2 +- go.sum | 4 ++++ src/internal/packager/images/pull.go | 2 +- src/test/e2e/36_custom_retries_test.go | 4 ++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 22f660f703..ee20ffb0c8 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/anchore/clio v0.0.0-20240307182142-fb5fc4c9db3c github.com/anchore/stereoscope v0.0.1 github.com/anchore/syft v0.100.0 - github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411154501-d8e8145af112 + github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416125436-7c54d5ed05a0 github.com/defenseunicorns/pkg/oci v0.0.1 github.com/derailed/k9s v0.31.7 github.com/distribution/reference v0.5.0 diff --git a/go.sum b/go.sum index d92cd77f9d..b242c2cf12 100644 --- a/go.sum +++ b/go.sum @@ -599,6 +599,10 @@ github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411133953-c27224059905 h1:x github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411133953-c27224059905/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411154501-d8e8145af112 h1:Z6MQCjpKBA1CQfr3XqaRQsjXt+LYtNzfT4Adm53mfyE= github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411154501-d8e8145af112/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= +github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416120542-1f7cefc0731b h1:0GwCv4pz9ncqXW/gDpGeKalXRVZwXmarPubett6GbwU= +github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416120542-1f7cefc0731b/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= +github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416125436-7c54d5ed05a0 h1:7Pzb4Y6f+GCeCVrgeMW/jD3X7qSGrxdQWeoyfskc4AU= +github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416125436-7c54d5ed05a0/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= github.com/defenseunicorns/pkg/oci v0.0.1 h1:EFRp3NeiwzhOWKpQ6mAxi0l9chnrAvDcIgjMr0o0fkM= github.com/defenseunicorns/pkg/oci v0.0.1/go.mod h1:zVBgRjckEAhfdvbnQrnfOP/3M/GYJkIgWtJtY7pjYdo= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M= diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 30d4d7820b..38a49e5956 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -317,7 +317,7 @@ func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Im } if strings.Contains(src, "@") && (idx.MediaType == ctypes.OCIImageIndex || idx.MediaType == ctypes.DockerManifestList) { - imageOptions := "please select one of the images below based on your platform" + imageOptions := "please select one of the images below based on your platform to use instead" imageBaseName := strings.Split(src, "@")[0] for _, manifest := range idx.Manifests { imageOptions = fmt.Sprintf("%s\n %s@%s for platform %s", imageOptions, imageBaseName, manifest.Digest, manifest.Platform) diff --git a/src/test/e2e/36_custom_retries_test.go b/src/test/e2e/36_custom_retries_test.go index e66a6ff435..53d877924f 100644 --- a/src/test/e2e/36_custom_retries_test.go +++ b/src/test/e2e/36_custom_retries_test.go @@ -27,7 +27,7 @@ func TestRetries(t *testing.T) { stdOut, stdErr, err = e2e.Zarf("package", "deploy", path.Join(tmpDir, pkgName), "--retries", "2", "--timeout", "3s", "--tmpdir", tmpDir, "--confirm") require.Error(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "Retrying (1/2) in 5s:") - require.Contains(t, stdErr, "Retrying (2/2) in 10s:") + require.Contains(t, stdErr, "Retrying in 5s:") + require.Contains(t, stdErr, "Retrying in 10s:") require.Contains(t, stdErr, "unable to install chart after 2 attempts") } From 9e29b74dd045efdc338d3401367482dceb76734f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 16 Apr 2024 13:52:10 +0000 Subject: [PATCH 09/38] deleting unnecessary check --- src/test/e2e/36_custom_retries_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/e2e/36_custom_retries_test.go b/src/test/e2e/36_custom_retries_test.go index 53d877924f..2bf046b9e3 100644 --- a/src/test/e2e/36_custom_retries_test.go +++ b/src/test/e2e/36_custom_retries_test.go @@ -28,6 +28,5 @@ func TestRetries(t *testing.T) { stdOut, stdErr, err = e2e.Zarf("package", "deploy", path.Join(tmpDir, pkgName), "--retries", "2", "--timeout", "3s", "--tmpdir", tmpDir, "--confirm") require.Error(t, err, stdOut, stdErr) require.Contains(t, stdErr, "Retrying in 5s:") - require.Contains(t, stdErr, "Retrying in 10s:") require.Contains(t, stdErr, "unable to install chart after 2 attempts") } From 1346af6f47386d385bb046b3e5cef5f5d38050f7 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 16 Apr 2024 13:53:41 +0000 Subject: [PATCH 10/38] fix test --- src/test/e2e/36_custom_retries_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/e2e/36_custom_retries_test.go b/src/test/e2e/36_custom_retries_test.go index 2bf046b9e3..a0f33b94ac 100644 --- a/src/test/e2e/36_custom_retries_test.go +++ b/src/test/e2e/36_custom_retries_test.go @@ -27,6 +27,6 @@ func TestRetries(t *testing.T) { stdOut, stdErr, err = e2e.Zarf("package", "deploy", path.Join(tmpDir, pkgName), "--retries", "2", "--timeout", "3s", "--tmpdir", tmpDir, "--confirm") require.Error(t, err, stdOut, stdErr) - require.Contains(t, stdErr, "Retrying in 5s:") + require.Contains(t, stdErr, "Retrying in 5s") require.Contains(t, stdErr, "unable to install chart after 2 attempts") } From ab508c268626d2f3290320815eb6c2d56d2d9bb6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 16 Apr 2024 15:04:12 +0000 Subject: [PATCH 11/38] changing how we determine if an image is an index list --- src/internal/packager/images/pull.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 38a49e5956..f0f4db892d 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -274,7 +274,7 @@ func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Im if err != nil { return nil, false, err } - } else if manifest, err := crane.Manifest(src, config.GetCraneOptions(i.Insecure)...); err != nil { + } else if desc, err := crane.Get(src, config.GetCraneOptions(i.Insecure)...); err != nil { // If crane is unable to pull the image, try to load it from the local docker daemon. message.Notef("Falling back to local 'docker' images, failed to find the manifest on a remote: %s", err.Error()) @@ -311,14 +311,17 @@ func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Im return nil, false, fmt.Errorf("failed to load image from docker daemon: %w", err) } } else { - var idx v1.IndexManifest - if err := json.Unmarshal(manifest, &idx); err != nil { - return nil, false, lang.ErrUnsupportedImageType - } - - if strings.Contains(src, "@") && (idx.MediaType == ctypes.OCIImageIndex || idx.MediaType == ctypes.DockerManifestList) { + if strings.Contains(src, "@") && (desc.MediaType == ctypes.OCIImageIndex || desc.MediaType == ctypes.DockerManifestList) { imageOptions := "please select one of the images below based on your platform to use instead" imageBaseName := strings.Split(src, "@")[0] + manifest, err := crane.Manifest(src, config.GetCraneOptions(i.Insecure)...) + if err != nil { + return nil, false, fmt.Errorf("%w: %w", lang.ErrUnsupportedImageType, err) + } + var idx v1.IndexManifest + if err := json.Unmarshal(manifest, &idx); err != nil { + return nil, false, fmt.Errorf("%w: %w", lang.ErrUnsupportedImageType, err) + } for _, manifest := range idx.Manifests { imageOptions = fmt.Sprintf("%s\n %s@%s for platform %s", imageOptions, imageBaseName, manifest.Digest, manifest.Platform) } From 1f5fe0db0ac4e6a5bd23b66e3eeb05f7a038af65 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 16 Apr 2024 15:11:23 +0000 Subject: [PATCH 12/38] test tablify --- src/test/e2e/14_create_sha_index_test.go | 29 +++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/test/e2e/14_create_sha_index_test.go b/src/test/e2e/14_create_sha_index_test.go index 4b564fc00e..9dee622c85 100644 --- a/src/test/e2e/14_create_sha_index_test.go +++ b/src/test/e2e/14_create_sha_index_test.go @@ -13,12 +13,29 @@ import ( func TestCreateIndexShaErrors(t *testing.T) { t.Log("E2E: CreateIndexShaErrors") - _, stderr, err := e2e.Zarf("package", "create", "src/test/packages/14-index-sha/image-index", "--confirm") - require.Error(t, err) - require.Contains(t, stderr, "ghcr.io/defenseunicorns/zarf/agent:v0.32.6@sha256:b3fabdc7d4ecd0f396016ef78da19002c39e3ace352ea0ae4baa2ce9d5958376") + testCases := []struct { + name string + packagePath string + expectedImageInStderr string + }{ + { + name: "Image Index", + packagePath: "src/test/packages/14-index-sha/image-index", + expectedImageInStderr: "ghcr.io/defenseunicorns/zarf/agent:v0.32.6@sha256:b3fabdc7d4ecd0f396016ef78da19002c39e3ace352ea0ae4baa2ce9d5958376", + }, + { + name: "Manifest List", + packagePath: "src/test/packages/14-index-sha/manifest-list", + expectedImageInStderr: "docker.io/defenseunicorns/zarf-game@sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff", + }, + } - _, stderr, err = e2e.Zarf("package", "create", "src/test/packages/14-index-sha/manifest-list", "--confirm") - require.Error(t, err) - require.Contains(t, stderr, "docker.io/defenseunicorns/zarf-game@sha256:f78e442f0f3eb3e9459b5ae6b1a8fda62f8dfe818112e7d130a4e8ae72b3cbff") + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, stderr, err := e2e.Zarf("package", "create", tc.packagePath, "--confirm") + require.Error(t, err) + require.Contains(t, stderr, tc.expectedImageInStderr) + }) + } } From d31acd673c86391bec62b8217773b7ba221d9c74 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 17 Apr 2024 12:40:43 +0000 Subject: [PATCH 13/38] change format --- src/internal/packager/images/pull.go | 17 +++++++++-------- src/test/e2e/14_create_sha_index_test.go | 2 +- src/test/packages/00-yq-checks/file1.yaml | 2 ++ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index f0f4db892d..d36b23fed4 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -311,17 +311,18 @@ func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Im return nil, false, fmt.Errorf("failed to load image from docker daemon: %w", err) } } else { - if strings.Contains(src, "@") && (desc.MediaType == ctypes.OCIImageIndex || desc.MediaType == ctypes.DockerManifestList) { - imageOptions := "please select one of the images below based on your platform to use instead" - imageBaseName := strings.Split(src, "@")[0] - manifest, err := crane.Manifest(src, config.GetCraneOptions(i.Insecure)...) - if err != nil { - return nil, false, fmt.Errorf("%w: %w", lang.ErrUnsupportedImageType, err) - } + reference, err := name.ParseReference(src) + if err != nil { + return nil, false, err + } + + if _, ok := reference.(name.Digest); ok && (desc.MediaType == ctypes.OCIImageIndex || desc.MediaType == ctypes.DockerManifestList) { var idx v1.IndexManifest - if err := json.Unmarshal(manifest, &idx); err != nil { + if err := json.Unmarshal(desc.Manifest, &idx); err != nil { return nil, false, fmt.Errorf("%w: %w", lang.ErrUnsupportedImageType, err) } + imageOptions := "please select one of the images below based on your platform to use instead" + imageBaseName := fmt.Sprintf("%s/%s", reference.Context().Registry.RegistryStr(), reference.Context().RepositoryStr()) for _, manifest := range idx.Manifests { imageOptions = fmt.Sprintf("%s\n %s@%s for platform %s", imageOptions, imageBaseName, manifest.Digest, manifest.Platform) } diff --git a/src/test/e2e/14_create_sha_index_test.go b/src/test/e2e/14_create_sha_index_test.go index 9dee622c85..cbb81348e5 100644 --- a/src/test/e2e/14_create_sha_index_test.go +++ b/src/test/e2e/14_create_sha_index_test.go @@ -21,7 +21,7 @@ func TestCreateIndexShaErrors(t *testing.T) { { name: "Image Index", packagePath: "src/test/packages/14-index-sha/image-index", - expectedImageInStderr: "ghcr.io/defenseunicorns/zarf/agent:v0.32.6@sha256:b3fabdc7d4ecd0f396016ef78da19002c39e3ace352ea0ae4baa2ce9d5958376", + expectedImageInStderr: "ghcr.io/defenseunicorns/zarf/agent@sha256:b3fabdc7d4ecd0f396016ef78da19002c39e3ace352ea0ae4baa2ce9d5958376", }, { name: "Manifest List", diff --git a/src/test/packages/00-yq-checks/file1.yaml b/src/test/packages/00-yq-checks/file1.yaml index f4256bbf42..e060e24286 100644 --- a/src/test/packages/00-yq-checks/file1.yaml +++ b/src/test/packages/00-yq-checks/file1.yaml @@ -1,3 +1,5 @@ items: - name: item1 - name: renamed-item + - name: item3 + - name: item4 From d83f1590fc84dfdada99b88cdc1694122d150651 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 17 Apr 2024 12:43:52 +0000 Subject: [PATCH 14/38] switching to refinfo --- src/internal/packager/images/pull.go | 9 ++++++--- src/test/e2e/14_create_sha_index_test.go | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index d36b23fed4..6e220fc1ac 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -311,18 +311,21 @@ func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Im return nil, false, fmt.Errorf("failed to load image from docker daemon: %w", err) } } else { - reference, err := name.ParseReference(src) + refInfo, err := transform.ParseImageRef(src) if err != nil { return nil, false, err } - if _, ok := reference.(name.Digest); ok && (desc.MediaType == ctypes.OCIImageIndex || desc.MediaType == ctypes.DockerManifestList) { + if refInfo.Digest != "" && (desc.MediaType == ctypes.OCIImageIndex || desc.MediaType == ctypes.DockerManifestList) { var idx v1.IndexManifest if err := json.Unmarshal(desc.Manifest, &idx); err != nil { return nil, false, fmt.Errorf("%w: %w", lang.ErrUnsupportedImageType, err) } imageOptions := "please select one of the images below based on your platform to use instead" - imageBaseName := fmt.Sprintf("%s/%s", reference.Context().Registry.RegistryStr(), reference.Context().RepositoryStr()) + imageBaseName := refInfo.Name + if refInfo.Tag != "" { + imageBaseName = fmt.Sprintf("%s:%s", imageBaseName, refInfo.Tag) + } for _, manifest := range idx.Manifests { imageOptions = fmt.Sprintf("%s\n %s@%s for platform %s", imageOptions, imageBaseName, manifest.Digest, manifest.Platform) } diff --git a/src/test/e2e/14_create_sha_index_test.go b/src/test/e2e/14_create_sha_index_test.go index cbb81348e5..9dee622c85 100644 --- a/src/test/e2e/14_create_sha_index_test.go +++ b/src/test/e2e/14_create_sha_index_test.go @@ -21,7 +21,7 @@ func TestCreateIndexShaErrors(t *testing.T) { { name: "Image Index", packagePath: "src/test/packages/14-index-sha/image-index", - expectedImageInStderr: "ghcr.io/defenseunicorns/zarf/agent@sha256:b3fabdc7d4ecd0f396016ef78da19002c39e3ace352ea0ae4baa2ce9d5958376", + expectedImageInStderr: "ghcr.io/defenseunicorns/zarf/agent:v0.32.6@sha256:b3fabdc7d4ecd0f396016ef78da19002c39e3ace352ea0ae4baa2ce9d5958376", }, { name: "Manifest List", From efd6a80b105df61d392bcc16aa494bfc74b4fd0e Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 17 Apr 2024 13:42:16 +0000 Subject: [PATCH 15/38] fix yq test : --- src/test/e2e/00_use_cli_test.go | 19 +++++++++++++++---- src/test/packages/00-yq-checks/file1.yaml | 2 -- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/test/e2e/00_use_cli_test.go b/src/test/e2e/00_use_cli_test.go index eec380619e..149abaecde 100644 --- a/src/test/e2e/00_use_cli_test.go +++ b/src/test/e2e/00_use_cli_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/defenseunicorns/pkg/helpers" + "github.com/otiai10/copy" "github.com/stretchr/testify/require" ) @@ -216,21 +217,31 @@ func TestUseCLI(t *testing.T) { t.Run("zarf tools yq should function appropriately across different uses", func(t *testing.T) { t.Parallel() - file := "src/test/packages/00-yq-checks/file1.yaml" - otherFile := "src/test/packages/00-yq-checks/file2.yaml" + tmpdir := t.TempDir() + originalPath := "src/test/packages/00-yq-checks" + + originalFile := filepath.Join(originalPath, "file1.yaml") + originalOtherFile := filepath.Join(originalPath, "file2.yaml") + + file := filepath.Join(tmpdir, "file1.yaml") + otherFile := filepath.Join(tmpdir, "file2.yaml") + + copy.Copy(originalFile, file) + copy.Copy(originalOtherFile, otherFile) // Test that yq can eval properly _, stdErr, err := e2e.Zarf("tools", "yq", "eval", "-i", `.items[1].name = "renamed-item"`, file) require.NoError(t, err, stdErr) - stdOut, stdErr, err := e2e.Zarf("tools", "yq", ".items[1].name", file) + stdOut, _, err := e2e.Zarf("tools", "yq", ".items[1].name", file) + require.NoError(t, err) require.Contains(t, stdOut, "renamed-item") // Test that yq ea can be used properly _, stdErr, err = e2e.Zarf("tools", "yq", "eval-all", "-i", `. as $doc ireduce ({}; .items += $doc.items)`, file, otherFile) require.NoError(t, err, stdErr) stdOut, stdErr, err = e2e.Zarf("tools", "yq", "e", ".items | length", file) + require.NoError(t, err) require.Equal(t, "4\n", stdOut) }) } - diff --git a/src/test/packages/00-yq-checks/file1.yaml b/src/test/packages/00-yq-checks/file1.yaml index e060e24286..f4256bbf42 100644 --- a/src/test/packages/00-yq-checks/file1.yaml +++ b/src/test/packages/00-yq-checks/file1.yaml @@ -1,5 +1,3 @@ items: - name: item1 - name: renamed-item - - name: item3 - - name: item4 From e16927f036f7d74256b1c2c79fa5e715f7799c11 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 17 Apr 2024 19:02:06 +0000 Subject: [PATCH 16/38] mod tidy --- go.mod | 2 +- go.sum | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/go.mod b/go.mod index ee20ffb0c8..e865724070 100644 --- a/go.mod +++ b/go.mod @@ -376,7 +376,7 @@ require ( github.com/opencontainers/selinux v1.11.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/openvex/go-vex v0.2.5 // indirect - github.com/otiai10/copy v1.14.0 // indirect + github.com/otiai10/copy v1.14.0 github.com/owenrumney/go-sarif v1.1.2-0.20231003122901-1000f5e05554 // indirect github.com/package-url/packageurl-go v0.1.1 // indirect github.com/pborman/indent v1.2.1 // indirect diff --git a/go.sum b/go.sum index b242c2cf12..8cc10c8457 100644 --- a/go.sum +++ b/go.sum @@ -593,14 +593,6 @@ github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c= github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6 h1:gwevOZ0fxT2nzM9hrtdPbsiOHjFqDRIYMzJHba3/G6Q= github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6/go.mod h1:StKLYMmPj1R5yIs6CK49EkcW1TvUYuw5Vri+LRk7Dy8= -github.com/defenseunicorns/pkg/helpers v1.0.0 h1:0o3Rs+J/g0UemZHcENBS1Z2Qw2y4FIUUrGs75iEyPb4= -github.com/defenseunicorns/pkg/helpers v1.0.0/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= -github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411133953-c27224059905 h1:x4KxchKd18SoYFAb/UmqpsJVhOVMB1VYJsVxt7w3s9s= -github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411133953-c27224059905/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= -github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411154501-d8e8145af112 h1:Z6MQCjpKBA1CQfr3XqaRQsjXt+LYtNzfT4Adm53mfyE= -github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240411154501-d8e8145af112/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= -github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416120542-1f7cefc0731b h1:0GwCv4pz9ncqXW/gDpGeKalXRVZwXmarPubett6GbwU= -github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416120542-1f7cefc0731b/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416125436-7c54d5ed05a0 h1:7Pzb4Y6f+GCeCVrgeMW/jD3X7qSGrxdQWeoyfskc4AU= github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416125436-7c54d5ed05a0/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= github.com/defenseunicorns/pkg/oci v0.0.1 h1:EFRp3NeiwzhOWKpQ6mAxi0l9chnrAvDcIgjMr0o0fkM= From d18b3d8c555535337072bc04200fcd335dc351b9 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 17 Apr 2024 19:20:26 +0000 Subject: [PATCH 17/38] yq test cleanup --- src/test/e2e/00_use_cli_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/test/e2e/00_use_cli_test.go b/src/test/e2e/00_use_cli_test.go index 149abaecde..49f3488588 100644 --- a/src/test/e2e/00_use_cli_test.go +++ b/src/test/e2e/00_use_cli_test.go @@ -226,8 +226,10 @@ func TestUseCLI(t *testing.T) { file := filepath.Join(tmpdir, "file1.yaml") otherFile := filepath.Join(tmpdir, "file2.yaml") - copy.Copy(originalFile, file) - copy.Copy(originalOtherFile, otherFile) + err := copy.Copy(originalFile, file) + require.NoError(t, err) + err = copy.Copy(originalOtherFile, otherFile) + require.NoError(t, err) // Test that yq can eval properly _, stdErr, err := e2e.Zarf("tools", "yq", "eval", "-i", `.items[1].name = "renamed-item"`, file) @@ -237,9 +239,9 @@ func TestUseCLI(t *testing.T) { require.Contains(t, stdOut, "renamed-item") // Test that yq ea can be used properly - _, stdErr, err = e2e.Zarf("tools", "yq", "eval-all", "-i", `. as $doc ireduce ({}; .items += $doc.items)`, file, otherFile) - require.NoError(t, err, stdErr) - stdOut, stdErr, err = e2e.Zarf("tools", "yq", "e", ".items | length", file) + _, _, err = e2e.Zarf("tools", "yq", "eval-all", "-i", `. as $doc ireduce ({}; .items += $doc.items)`, file, otherFile) + require.NoError(t, err) + stdOut, _, err = e2e.Zarf("tools", "yq", "e", ".items | length", file) require.NoError(t, err) require.Equal(t, "4\n", stdOut) From b61cbde1147a295d9aa4e1cc6eba28e4e62c2d59 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Wed, 17 Apr 2024 19:27:19 +0000 Subject: [PATCH 18/38] comment --- src/internal/packager/images/pull.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index 6e220fc1ac..ffa88b69f3 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -315,7 +315,7 @@ func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Im if err != nil { return nil, false, err } - + // Check if we have an indexImage or manifest list and if so error out if refInfo.Digest != "" && (desc.MediaType == ctypes.OCIImageIndex || desc.MediaType == ctypes.DockerManifestList) { var idx v1.IndexManifest if err := json.Unmarshal(desc.Manifest, &idx); err != nil { From b538c55edd8481bf1ff99a2114f85ee819521a82 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 16:06:44 +0000 Subject: [PATCH 19/38] use helpers release --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e865724070..3b729a8542 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/anchore/clio v0.0.0-20240307182142-fb5fc4c9db3c github.com/anchore/stereoscope v0.0.1 github.com/anchore/syft v0.100.0 - github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416125436-7c54d5ed05a0 + github.com/defenseunicorns/pkg/helpers v1.1.0 github.com/defenseunicorns/pkg/oci v0.0.1 github.com/derailed/k9s v0.31.7 github.com/distribution/reference v0.5.0 diff --git a/go.sum b/go.sum index 8cc10c8457..5b62a90774 100644 --- a/go.sum +++ b/go.sum @@ -595,6 +595,8 @@ github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6 h1:gw github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6/go.mod h1:StKLYMmPj1R5yIs6CK49EkcW1TvUYuw5Vri+LRk7Dy8= github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416125436-7c54d5ed05a0 h1:7Pzb4Y6f+GCeCVrgeMW/jD3X7qSGrxdQWeoyfskc4AU= github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416125436-7c54d5ed05a0/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= +github.com/defenseunicorns/pkg/helpers v1.1.0 h1:VU8Cr3IGFEDuZrfavxmo6YXsh5FZXhehy4clKXrHNGk= +github.com/defenseunicorns/pkg/helpers v1.1.0/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= github.com/defenseunicorns/pkg/oci v0.0.1 h1:EFRp3NeiwzhOWKpQ6mAxi0l9chnrAvDcIgjMr0o0fkM= github.com/defenseunicorns/pkg/oci v0.0.1/go.mod h1:zVBgRjckEAhfdvbnQrnfOP/3M/GYJkIgWtJtY7pjYdo= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M= From a3aa14d4862d3ef89b254a339faee3df3bc870c6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 16:33:21 +0000 Subject: [PATCH 20/38] new compare workflow --- .github/workflows/compare-cves.yml | 28 ++++++++++++++++++++++++++++ hack/.templates/compare.tmpl | 7 +++++++ hack/check-vulnerabilities.sh | 24 ++++++++++++++++++++++++ 3 files changed, 59 insertions(+) create mode 100644 .github/workflows/compare-cves.yml create mode 100644 hack/.templates/compare.tmpl create mode 100755 hack/check-vulnerabilities.sh diff --git a/.github/workflows/compare-cves.yml b/.github/workflows/compare-cves.yml new file mode 100644 index 0000000000..a940691ea5 --- /dev/null +++ b/.github/workflows/compare-cves.yml @@ -0,0 +1,28 @@ +name: Compare CVEs to main + +permissions: + contents: read + +on: + pull_request: + paths: + - "go.mod" + - "go.sum" + - "cargo.toml" + - "cargo.lock" + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Setup golang + uses: ./.github/actions/golang + + - name: Install tools + uses: ./.github/actions/install-tools + + - name: Check for CVEs in Dependencies + run: "hack/check-vulnerabilities.sh" diff --git a/hack/.templates/compare.tmpl b/hack/.templates/compare.tmpl new file mode 100644 index 0000000000..763849a52e --- /dev/null +++ b/hack/.templates/compare.tmpl @@ -0,0 +1,7 @@ +[ + {{- $length := len .Matches -}} + {{- range $index, $match := .Matches -}} + "{{$match.Vulnerability.ID}}" + {{ if lt (add $index 1) $length }},{{ end }} + {{- end -}} +] diff --git a/hack/check-vulnerabilities.sh b/hack/check-vulnerabilities.sh new file mode 100755 index 0000000000..2280752117 --- /dev/null +++ b/hack/check-vulnerabilities.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# # Exit on error +set -e + +MAIN_BRANCH="main" +TARGET_BRANCH=$(git rev-parse --abbrev-ref HEAD) + +git checkout $MAIN_BRANCH +zarf tools sbom scan . -o json --exclude './site' --exclude './examples' | grype -o template -t hack/.templates/compare.tmpl > build/main.json + +git checkout $TARGET_BRANCH +zarf tools sbom scan . -o json --exclude './site' --exclude './examples' | grype -o template -t hack/.templates/compare.tmpl > build/target.json + + +result=$(jq --slurp '.[0] - .[1]' build/target.json build/main.json) + +if [[ "$result" == "[]" ]]; then + echo "no new vulnerabilities on $TARGET_BRANCH" + exit 0 +else + echo "new CVEs have been added with IDs $result" + exit 1 +fi From 22b0817591614fa769ddaa881d0ee11a511caa73 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 16:33:59 +0000 Subject: [PATCH 21/38] script header --- hack/check-vulnerabilities.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hack/check-vulnerabilities.sh b/hack/check-vulnerabilities.sh index 2280752117..81fa4f04ca 100755 --- a/hack/check-vulnerabilities.sh +++ b/hack/check-vulnerabilities.sh @@ -1,7 +1,7 @@ -#!/bin/bash +#!/usr/bin/env bash + +set -euo pipefail -# # Exit on error -set -e MAIN_BRANCH="main" TARGET_BRANCH=$(git rev-parse --abbrev-ref HEAD) From feec05211ebca0d8aa0dc8f717d4634a8215d62f Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 16:37:50 +0000 Subject: [PATCH 22/38] checkout main --- .github/workflows/compare-cves.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/compare-cves.yml b/.github/workflows/compare-cves.yml index a940691ea5..db7f853e07 100644 --- a/.github/workflows/compare-cves.yml +++ b/.github/workflows/compare-cves.yml @@ -15,9 +15,14 @@ jobs: validate: runs-on: ubuntu-latest steps: - - name: Checkout + - name: Checkout repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Checkout main + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: main + - name: Setup golang uses: ./.github/actions/golang From 4b50f6b30e4faa95b1763804bf8ac6e7b043e268 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 16:39:17 +0000 Subject: [PATCH 23/38] fetch-depth 0 --- .github/workflows/compare-cves.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/compare-cves.yml b/.github/workflows/compare-cves.yml index db7f853e07..dab4a6f4bb 100644 --- a/.github/workflows/compare-cves.yml +++ b/.github/workflows/compare-cves.yml @@ -17,11 +17,8 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - - name: Checkout main - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - ref: main + fetch-depth: 0 - name: Setup golang uses: ./.github/actions/golang From 91f54019e5853fb5ef8cae26cb167adc47d90455 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 16:40:46 +0000 Subject: [PATCH 24/38] fetch main --- .github/workflows/compare-cves.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/compare-cves.yml b/.github/workflows/compare-cves.yml index dab4a6f4bb..326597a74c 100644 --- a/.github/workflows/compare-cves.yml +++ b/.github/workflows/compare-cves.yml @@ -17,8 +17,9 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - fetch-depth: 0 + + - name: fetch main + run: git fetch origin master --depth 1 - name: Setup golang uses: ./.github/actions/golang From 81316563eb9004d0fca7c553398fd487b9a89902 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 16:41:59 +0000 Subject: [PATCH 25/38] whoops --- .github/workflows/compare-cves.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compare-cves.yml b/.github/workflows/compare-cves.yml index 326597a74c..359ebe27eb 100644 --- a/.github/workflows/compare-cves.yml +++ b/.github/workflows/compare-cves.yml @@ -19,7 +19,7 @@ jobs: uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - name: fetch main - run: git fetch origin master --depth 1 + run: git fetch origin main --depth 1 - name: Setup golang uses: ./.github/actions/golang From 6f9346bfb4a460950c6b419e03c8a84b4484676d Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 16:43:42 +0000 Subject: [PATCH 26/38] check --- hack/check-vulnerabilities.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hack/check-vulnerabilities.sh b/hack/check-vulnerabilities.sh index 81fa4f04ca..459c0db587 100755 --- a/hack/check-vulnerabilities.sh +++ b/hack/check-vulnerabilities.sh @@ -7,10 +7,10 @@ MAIN_BRANCH="main" TARGET_BRANCH=$(git rev-parse --abbrev-ref HEAD) git checkout $MAIN_BRANCH -zarf tools sbom scan . -o json --exclude './site' --exclude './examples' | grype -o template -t hack/.templates/compare.tmpl > build/main.json +go run main.go tools sbom scan . -o json --exclude './site' --exclude './examples' | grype -o template -t hack/.templates/compare.tmpl > build/main.json git checkout $TARGET_BRANCH -zarf tools sbom scan . -o json --exclude './site' --exclude './examples' | grype -o template -t hack/.templates/compare.tmpl > build/target.json +go run main.go tools sbom scan . -o json --exclude './site' --exclude './examples' | grype -o template -t hack/.templates/compare.tmpl > build/target.json result=$(jq --slurp '.[0] - .[1]' build/target.json build/main.json) From 66a2e52a64a6341c094953f17034bd7786b8039a Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 17:01:44 +0000 Subject: [PATCH 27/38] check --- hack/check-vulnerabilities.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hack/check-vulnerabilities.sh b/hack/check-vulnerabilities.sh index 459c0db587..ec6db589fa 100755 --- a/hack/check-vulnerabilities.sh +++ b/hack/check-vulnerabilities.sh @@ -6,6 +6,8 @@ set -euo pipefail MAIN_BRANCH="main" TARGET_BRANCH=$(git rev-parse --abbrev-ref HEAD) +mkdir -p build + git checkout $MAIN_BRANCH go run main.go tools sbom scan . -o json --exclude './site' --exclude './examples' | grype -o template -t hack/.templates/compare.tmpl > build/main.json From c6230b73b74de45f30ea6056f78eae676b555ba6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 17:08:27 +0000 Subject: [PATCH 28/38] check vulnerabilities --- hack/check-vulnerabilities.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hack/check-vulnerabilities.sh b/hack/check-vulnerabilities.sh index ec6db589fa..7db9cd2a1a 100755 --- a/hack/check-vulnerabilities.sh +++ b/hack/check-vulnerabilities.sh @@ -9,9 +9,10 @@ TARGET_BRANCH=$(git rev-parse --abbrev-ref HEAD) mkdir -p build git checkout $MAIN_BRANCH -go run main.go tools sbom scan . -o json --exclude './site' --exclude './examples' | grype -o template -t hack/.templates/compare.tmpl > build/main.json +go run main.go tools sbom scan . -o json --exclude './site' --exclude './examples' > build/main-syft.json git checkout $TARGET_BRANCH +cat main-syft.json | grype -o template -t hack/.templates/compare.tmpl > build/main.json go run main.go tools sbom scan . -o json --exclude './site' --exclude './examples' | grype -o template -t hack/.templates/compare.tmpl > build/target.json From 355fd636b62923b24bd59f37c28e0bcf2162ec16 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 17:09:07 +0000 Subject: [PATCH 29/38] main-syft.json --- hack/check-vulnerabilities.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/check-vulnerabilities.sh b/hack/check-vulnerabilities.sh index 7db9cd2a1a..1acccc3aa1 100755 --- a/hack/check-vulnerabilities.sh +++ b/hack/check-vulnerabilities.sh @@ -12,7 +12,7 @@ git checkout $MAIN_BRANCH go run main.go tools sbom scan . -o json --exclude './site' --exclude './examples' > build/main-syft.json git checkout $TARGET_BRANCH -cat main-syft.json | grype -o template -t hack/.templates/compare.tmpl > build/main.json +cat build/main-syft.json | grype -o template -t hack/.templates/compare.tmpl > build/main.json go run main.go tools sbom scan . -o json --exclude './site' --exclude './examples' | grype -o template -t hack/.templates/compare.tmpl > build/target.json From b14a4c21c73eb921b5a6e096fdb3d8d261d68de2 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 17:12:39 +0000 Subject: [PATCH 30/38] check --- hack/check-vulnerabilities.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hack/check-vulnerabilities.sh b/hack/check-vulnerabilities.sh index 1acccc3aa1..a13d1dc64d 100755 --- a/hack/check-vulnerabilities.sh +++ b/hack/check-vulnerabilities.sh @@ -5,6 +5,7 @@ set -euo pipefail MAIN_BRANCH="main" TARGET_BRANCH=$(git rev-parse --abbrev-ref HEAD) +echo "target branch is $TARGET_BRANCH" mkdir -p build From 57088c5e798d24744d56a25efb3ba20f72c0c005 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 17:16:45 +0000 Subject: [PATCH 31/38] check --- .github/workflows/compare-cves.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/compare-cves.yml b/.github/workflows/compare-cves.yml index 359ebe27eb..d5ade948a7 100644 --- a/.github/workflows/compare-cves.yml +++ b/.github/workflows/compare-cves.yml @@ -17,6 +17,8 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: "$GITHUB_REF_NAME" - name: fetch main run: git fetch origin main --depth 1 From e86f8e538c87a71ac1b2e398cbe85ca7b5969b76 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 17:17:51 +0000 Subject: [PATCH 32/38] check --- .github/workflows/compare-cves.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/compare-cves.yml b/.github/workflows/compare-cves.yml index d5ade948a7..e001eb8f20 100644 --- a/.github/workflows/compare-cves.yml +++ b/.github/workflows/compare-cves.yml @@ -18,7 +18,7 @@ jobs: - name: Checkout repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - ref: "$GITHUB_REF_NAME" + ref: ${{ github.ref_name }} - name: fetch main run: git fetch origin main --depth 1 From 6cfd0368ecafa5b8371a5f451e9cae722008b7e5 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 17:20:16 +0000 Subject: [PATCH 33/38] check --- .github/workflows/compare-cves.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/compare-cves.yml b/.github/workflows/compare-cves.yml index e001eb8f20..4ee0c8b1a3 100644 --- a/.github/workflows/compare-cves.yml +++ b/.github/workflows/compare-cves.yml @@ -17,8 +17,10 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} with: - ref: ${{ github.ref_name }} + ref: ${{ env.BRANCH_NAME }} - name: fetch main run: git fetch origin main --depth 1 From 4d2589689b02accfc4d97f280b51bd469ae68331 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Fri, 19 Apr 2024 17:21:03 +0000 Subject: [PATCH 34/38] check --- .github/workflows/compare-cves.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/compare-cves.yml b/.github/workflows/compare-cves.yml index 4ee0c8b1a3..f4c8500d88 100644 --- a/.github/workflows/compare-cves.yml +++ b/.github/workflows/compare-cves.yml @@ -17,10 +17,8 @@ jobs: steps: - name: Checkout repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - env: - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} with: - ref: ${{ env.BRANCH_NAME }} + ref: ${{ github.head_ref || github.ref_name }} - name: fetch main run: git fetch origin main --depth 1 From 35f189bfb170825aeb09e9fbfe8be235fe323e10 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 22 Apr 2024 15:33:02 +0000 Subject: [PATCH 35/38] fix comment --- src/internal/packager/images/pull.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/internal/packager/images/pull.go b/src/internal/packager/images/pull.go index ffa88b69f3..2db0200d53 100644 --- a/src/internal/packager/images/pull.go +++ b/src/internal/packager/images/pull.go @@ -315,7 +315,7 @@ func (i *ImageConfig) PullImage(src string, spinner *message.Spinner) (img v1.Im if err != nil { return nil, false, err } - // Check if we have an indexImage or manifest list and if so error out + // Check if we have an image index or manifest list and if so error out if refInfo.Digest != "" && (desc.MediaType == ctypes.OCIImageIndex || desc.MediaType == ctypes.DockerManifestList) { var idx v1.IndexManifest if err := json.Unmarshal(desc.Manifest, &idx); err != nil { From 74d7f52f1d7e57cafb7c2eb40120743b88530b07 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Mon, 22 Apr 2024 19:23:54 +0000 Subject: [PATCH 36/38] removing comapre cve things --- .github/workflows/compare-cves.yml | 33 ------------------------------ hack/.templates/compare.tmpl | 7 ------- hack/check-vulnerabilities.sh | 28 ------------------------- 3 files changed, 68 deletions(-) delete mode 100644 .github/workflows/compare-cves.yml delete mode 100644 hack/.templates/compare.tmpl delete mode 100755 hack/check-vulnerabilities.sh diff --git a/.github/workflows/compare-cves.yml b/.github/workflows/compare-cves.yml deleted file mode 100644 index f4c8500d88..0000000000 --- a/.github/workflows/compare-cves.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Compare CVEs to main - -permissions: - contents: read - -on: - pull_request: - paths: - - "go.mod" - - "go.sum" - - "cargo.toml" - - "cargo.lock" - -jobs: - validate: - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - ref: ${{ github.head_ref || github.ref_name }} - - - name: fetch main - run: git fetch origin main --depth 1 - - - name: Setup golang - uses: ./.github/actions/golang - - - name: Install tools - uses: ./.github/actions/install-tools - - - name: Check for CVEs in Dependencies - run: "hack/check-vulnerabilities.sh" diff --git a/hack/.templates/compare.tmpl b/hack/.templates/compare.tmpl deleted file mode 100644 index 763849a52e..0000000000 --- a/hack/.templates/compare.tmpl +++ /dev/null @@ -1,7 +0,0 @@ -[ - {{- $length := len .Matches -}} - {{- range $index, $match := .Matches -}} - "{{$match.Vulnerability.ID}}" - {{ if lt (add $index 1) $length }},{{ end }} - {{- end -}} -] diff --git a/hack/check-vulnerabilities.sh b/hack/check-vulnerabilities.sh deleted file mode 100755 index a13d1dc64d..0000000000 --- a/hack/check-vulnerabilities.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - - -MAIN_BRANCH="main" -TARGET_BRANCH=$(git rev-parse --abbrev-ref HEAD) -echo "target branch is $TARGET_BRANCH" - -mkdir -p build - -git checkout $MAIN_BRANCH -go run main.go tools sbom scan . -o json --exclude './site' --exclude './examples' > build/main-syft.json - -git checkout $TARGET_BRANCH -cat build/main-syft.json | grype -o template -t hack/.templates/compare.tmpl > build/main.json -go run main.go tools sbom scan . -o json --exclude './site' --exclude './examples' | grype -o template -t hack/.templates/compare.tmpl > build/target.json - - -result=$(jq --slurp '.[0] - .[1]' build/target.json build/main.json) - -if [[ "$result" == "[]" ]]; then - echo "no new vulnerabilities on $TARGET_BRANCH" - exit 0 -else - echo "new CVEs have been added with IDs $result" - exit 1 -fi From d4d5121628cbfdb16e25eb6ce0f469c5c3f078e6 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Tue, 23 Apr 2024 11:48:59 +0000 Subject: [PATCH 37/38] yq whitespace --- site/src/content/docs/commands/zarf_tools_yq_eval.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/site/src/content/docs/commands/zarf_tools_yq_eval.md b/site/src/content/docs/commands/zarf_tools_yq_eval.md index 09296535c4..215184cf00 100644 --- a/site/src/content/docs/commands/zarf_tools_yq_eval.md +++ b/site/src/content/docs/commands/zarf_tools_yq_eval.md @@ -12,11 +12,11 @@ tableOfContents: false ### Synopsis -yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) +yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples. ## Evaluate Sequence ## -This command iterates over each yaml document from each given file, applies the +This command iterates over each yaml document from each given file, applies the expression and prints the result in sequence. ``` @@ -98,3 +98,4 @@ zarf tools yq e '.a.b = "cool"' -i file.yaml ### SEE ALSO * [zarf tools yq](/commands/zarf_tools_yq/) - yq is a lightweight and portable command-line data file processor. + From 4420e1d68f1b3a2dceff7ff113954faea46a9175 Mon Sep 17 00:00:00 2001 From: Austin Abro Date: Thu, 25 Apr 2024 17:19:16 +0000 Subject: [PATCH 38/38] mod tidy --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e906ff8abf..4be2e73cf0 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/anchore/clio v0.0.0-20240307182142-fb5fc4c9db3c github.com/anchore/stereoscope v0.0.1 github.com/anchore/syft v0.100.0 - github.com/defenseunicorns/pkg/helpers v1.1.0 + github.com/defenseunicorns/pkg/helpers v1.1.1 github.com/defenseunicorns/pkg/oci v0.0.1 github.com/derailed/k9s v0.31.7 github.com/distribution/reference v0.5.0 diff --git a/go.sum b/go.sum index 8d8227227b..a45dbd3c96 100644 --- a/go.sum +++ b/go.sum @@ -595,10 +595,10 @@ github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c= github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6 h1:gwevOZ0fxT2nzM9hrtdPbsiOHjFqDRIYMzJHba3/G6Q= github.com/defenseunicorns/gojsonschema v0.0.0-20231116163348-e00f069122d6/go.mod h1:StKLYMmPj1R5yIs6CK49EkcW1TvUYuw5Vri+LRk7Dy8= -github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416125436-7c54d5ed05a0 h1:7Pzb4Y6f+GCeCVrgeMW/jD3X7qSGrxdQWeoyfskc4AU= -github.com/defenseunicorns/pkg/helpers v1.0.1-0.20240416125436-7c54d5ed05a0/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= github.com/defenseunicorns/pkg/helpers v1.1.0 h1:VU8Cr3IGFEDuZrfavxmo6YXsh5FZXhehy4clKXrHNGk= github.com/defenseunicorns/pkg/helpers v1.1.0/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= +github.com/defenseunicorns/pkg/helpers v1.1.1 h1:p3pKeK5SeFaoZUJZIX9sEsJqX1CGGMS8OpQMPgJtSqM= +github.com/defenseunicorns/pkg/helpers v1.1.1/go.mod h1:F4S5VZLDrlNWQKklzv4v9tFWjjZNhxJ1gT79j4XiLwk= github.com/defenseunicorns/pkg/oci v0.0.1 h1:EFRp3NeiwzhOWKpQ6mAxi0l9chnrAvDcIgjMr0o0fkM= github.com/defenseunicorns/pkg/oci v0.0.1/go.mod h1:zVBgRjckEAhfdvbnQrnfOP/3M/GYJkIgWtJtY7pjYdo= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M=