From 7a6241970fb390896a424c82d8a6c196594c5492 Mon Sep 17 00:00:00 2001 From: unclegedd Date: Wed, 29 May 2024 08:38:12 -0500 Subject: [PATCH 1/5] fix: ensure partial pkg is correct --- src/pkg/sources/tarball.go | 6 +++++- src/test/bundles/14-optional-components/uds-bundle.yaml | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/pkg/sources/tarball.go b/src/pkg/sources/tarball.go index 0917aa5a..08fdf921 100644 --- a/src/pkg/sources/tarball.go +++ b/src/pkg/sources/tarball.go @@ -61,6 +61,10 @@ func (t *TarballBundle) LoadPackage(dst *layout.PackagePaths, filter filters.Com return zarfTypes.ZarfPackage{}, nil, nil } + // grab number of components before filtering to determine if package is partial + numComponents := len(pkg.Components) + + // filter components based on required + specified optional components pkg.Components, err = filter.Apply(pkg) if err != nil { return pkg, nil, err @@ -68,7 +72,7 @@ func (t *TarballBundle) LoadPackage(dst *layout.PackagePaths, filter filters.Com dst.SetFromPaths(files) - isPartialPkg := t.PkgOpts.OptionalComponents != "" + isPartialPkg := t.PkgOpts.OptionalComponents != "" || numComponents > len(pkg.Components) if err := sources.ValidatePackageIntegrity(dst, pkg.Metadata.AggregateChecksum, isPartialPkg); err != nil { return zarfTypes.ZarfPackage{}, nil, err diff --git a/src/test/bundles/14-optional-components/uds-bundle.yaml b/src/test/bundles/14-optional-components/uds-bundle.yaml index e9486981..d43abcd3 100644 --- a/src/test/bundles/14-optional-components/uds-bundle.yaml +++ b/src/test/bundles/14-optional-components/uds-bundle.yaml @@ -8,8 +8,8 @@ packages: - name: prometheus repository: localhost:888/prometheus ref: 0.0.1 - optionalComponents: - - upload-image +# optionalComponents: +# - upload-image - name: podinfo-and-nginx path: ../../packages/podinfo-and-nginx From 9b65caeb7fb0a04f2e2d77cfa807c0b332b8936b Mon Sep 17 00:00:00 2001 From: unclegedd Date: Wed, 29 May 2024 09:37:26 -0500 Subject: [PATCH 2/5] WIP: test... --- .../14-optional-components/uds-bundle.yaml | 21 ++++++++++++------- .../packages/podinfo-nginx-apache/apache.yaml | 0 .../zarf.yaml | 13 +++++++++++- 3 files changed, 26 insertions(+), 8 deletions(-) create mode 100644 src/test/packages/podinfo-nginx-apache/apache.yaml rename src/test/packages/{podinfo-and-nginx => podinfo-nginx-apache}/zarf.yaml (57%) diff --git a/src/test/bundles/14-optional-components/uds-bundle.yaml b/src/test/bundles/14-optional-components/uds-bundle.yaml index d43abcd3..0e12a478 100644 --- a/src/test/bundles/14-optional-components/uds-bundle.yaml +++ b/src/test/bundles/14-optional-components/uds-bundle.yaml @@ -5,14 +5,21 @@ metadata: version: 0.0.1 packages: - - name: prometheus - repository: localhost:888/prometheus + # deploys apache as a required component (no optionalComponents key) + - name: podinfo-nginx-apache + path: ../../packages/podinfo-nginx-apache ref: 0.0.1 + + # deploys prometheus as a required component and upload-image as an optional component (with noOptionalComponents key) +# - name: prometheus +# repository: localhost:888/prometheus +# ref: 0.0.1 # optionalComponents: # - upload-image - - name: podinfo-and-nginx - path: ../../packages/podinfo-and-nginx - ref: 0.0.1 - optionalComponents: - - podinfo + # deploys podinfo as an optional component and apache as a required component +# - name: podinfo-nginx-apache +# path: ../../packages/podinfo-nginx-apache +# ref: 0.0.1 +# optionalComponents: +# - podinfo diff --git a/src/test/packages/podinfo-nginx-apache/apache.yaml b/src/test/packages/podinfo-nginx-apache/apache.yaml new file mode 100644 index 00000000..e69de29b diff --git a/src/test/packages/podinfo-and-nginx/zarf.yaml b/src/test/packages/podinfo-nginx-apache/zarf.yaml similarity index 57% rename from src/test/packages/podinfo-and-nginx/zarf.yaml rename to src/test/packages/podinfo-nginx-apache/zarf.yaml index 09c8c62e..4ab2da3c 100644 --- a/src/test/packages/podinfo-and-nginx/zarf.yaml +++ b/src/test/packages/podinfo-nginx-apache/zarf.yaml @@ -1,10 +1,21 @@ kind: ZarfPackageConfig metadata: - name: podinfo-and-nginx + name: podinfo-nginx-apache description: used to test bundles with optional components version: 0.0.1 components: + # only req'd component in this pkg + - name: apache + required: true + manifests: + - name: manifests + namespace: httpbin + files: + - "httpbin.yaml" + images: + - docker.io/kong/httpbin:latest + - name: podinfo import: path: ../podinfo From f6a6865822eb7677ebb11edc826c04da98902665 Mon Sep 17 00:00:00 2001 From: unclegedd Date: Wed, 29 May 2024 09:37:42 -0500 Subject: [PATCH 3/5] WIP: test... --- .github/workflows/nightly-uds-core.yaml | 14 ++-------- .github/workflows/release.yaml | 4 +++ src/pkg/sources/common.go | 11 ++++++++ src/pkg/sources/remote.go | 8 +++--- src/pkg/sources/tarball.go | 12 +++----- .../14-optional-components/uds-bundle.yaml | 28 +++++++++---------- src/test/e2e/optional_bundle_test.go | 11 ++++---- .../packages/podinfo-nginx-apache/apache.yaml | 0 src/test/packages/podinfo-nginx/apache.yaml | 21 ++++++++++++++ .../zarf.yaml | 18 ++++++------ tasks/tests.yaml | 9 ++++++ 11 files changed, 84 insertions(+), 52 deletions(-) delete mode 100644 src/test/packages/podinfo-nginx-apache/apache.yaml create mode 100644 src/test/packages/podinfo-nginx/apache.yaml rename src/test/packages/{podinfo-nginx-apache => podinfo-nginx}/zarf.yaml (68%) diff --git a/.github/workflows/nightly-uds-core.yaml b/.github/workflows/nightly-uds-core.yaml index 57de7c1b..e626d477 100644 --- a/.github/workflows/nightly-uds-core.yaml +++ b/.github/workflows/nightly-uds-core.yaml @@ -37,18 +37,10 @@ jobs: run: | chmod +x build/uds - - name: install-k3d + - name: Install-k3d run: "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash" shell: bash - - name: Deploy UDS Core bundle - # renovate: datasource=github-tags depName=defenseunicorns/uds-core versioning=semver - run: build/uds deploy k3d-core-istio-dev:0.16.1 --confirm - shell: bash - - - name: Validate UDS Core deployment - run: | - build/uds zarf tools wait-for gateway admin-gateway -n istio-admin-gateway --timeout 10s - build/uds zarf tools wait-for gateway passthrough-gateway -n istio-passthrough-gateway --timeout 10s - build/uds zarf tools wait-for gateway tenant-gateway -n istio-tenant-gateway --timeout 10s + - name: Run UDS Core smoke test + run: build/uds run test:ci-uds-core-smoke-test shell: bash diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 3e1e6738..2cf61a11 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -79,6 +79,10 @@ jobs: env: GITHUB_TOKEN: secrets.GITHUB_TOKEN + - name: Run UDS Core smoke test + run: build/uds run test:ci-uds-core-smoke-test + shell: bash + - name: Save logs if: always() uses: ./.github/actions/save-logs diff --git a/src/pkg/sources/common.go b/src/pkg/sources/common.go index 7f47b252..d302af2a 100644 --- a/src/pkg/sources/common.go +++ b/src/pkg/sources/common.go @@ -5,6 +5,7 @@ package sources import ( + "github.com/defenseunicorns/zarf/src/pkg/packager/filters" zarfTypes "github.com/defenseunicorns/zarf/src/types" ) @@ -33,3 +34,13 @@ func setAsYOLO(pkg *zarfTypes.ZarfPackage) { pkg.Components[idx].Repos = []string{} } } + +// handleFilter filters components and checks if a package is a partial package by checking its number of components +func handleFilter(pkg zarfTypes.ZarfPackage, filter filters.ComponentFilterStrategy) ([]zarfTypes.ZarfComponent, bool, error) { + numComponents := len(pkg.Components) + filteredComps, err := filter.Apply(pkg) + if err != nil { + return nil, false, err + } + return filteredComps, numComponents > len(filteredComps), nil +} diff --git a/src/pkg/sources/remote.go b/src/pkg/sources/remote.go index 825bc616..7dc30537 100644 --- a/src/pkg/sources/remote.go +++ b/src/pkg/sources/remote.go @@ -57,15 +57,15 @@ func (r *RemoteBundle) LoadPackage(dst *layout.PackagePaths, filter filters.Comp return zarfTypes.ZarfPackage{}, nil, nil } - pkg.Components, err = filter.Apply(pkg) + // filter pkg components and determine if its a partial pkg + filteredComps, isPartialPkg, err := handleFilter(pkg, filter) if err != nil { - return pkg, nil, err + return zarfTypes.ZarfPackage{}, nil, err } + pkg.Components = filteredComps dst.SetFromLayers(layers) - isPartialPkg := r.PkgOpts.OptionalComponents != "" - err = sources.ValidatePackageIntegrity(dst, pkg.Metadata.AggregateChecksum, isPartialPkg) if err != nil { return zarfTypes.ZarfPackage{}, nil, err diff --git a/src/pkg/sources/tarball.go b/src/pkg/sources/tarball.go index 08fdf921..051c3256 100644 --- a/src/pkg/sources/tarball.go +++ b/src/pkg/sources/tarball.go @@ -61,19 +61,15 @@ func (t *TarballBundle) LoadPackage(dst *layout.PackagePaths, filter filters.Com return zarfTypes.ZarfPackage{}, nil, nil } - // grab number of components before filtering to determine if package is partial - numComponents := len(pkg.Components) - - // filter components based on required + specified optional components - pkg.Components, err = filter.Apply(pkg) + // filter pkg components and determine if its a partial pkg + filteredComps, isPartialPkg, err := handleFilter(pkg, filter) if err != nil { - return pkg, nil, err + return zarfTypes.ZarfPackage{}, nil, err } + pkg.Components = filteredComps dst.SetFromPaths(files) - isPartialPkg := t.PkgOpts.OptionalComponents != "" || numComponents > len(pkg.Components) - if err := sources.ValidatePackageIntegrity(dst, pkg.Metadata.AggregateChecksum, isPartialPkg); err != nil { return zarfTypes.ZarfPackage{}, nil, err } diff --git a/src/test/bundles/14-optional-components/uds-bundle.yaml b/src/test/bundles/14-optional-components/uds-bundle.yaml index 0e12a478..d41549af 100644 --- a/src/test/bundles/14-optional-components/uds-bundle.yaml +++ b/src/test/bundles/14-optional-components/uds-bundle.yaml @@ -5,21 +5,21 @@ metadata: version: 0.0.1 packages: - # deploys apache as a required component (no optionalComponents key) - - name: podinfo-nginx-apache - path: ../../packages/podinfo-nginx-apache - ref: 0.0.1 + - name: init + repository: ghcr.io/defenseunicorns/packages/init + # renovate: datasource=github-tags depName=defenseunicorns/zarf + ref: v0.34.0 # deploys prometheus as a required component and upload-image as an optional component (with noOptionalComponents key) -# - name: prometheus -# repository: localhost:888/prometheus -# ref: 0.0.1 -# optionalComponents: -# - upload-image + - name: prometheus + repository: localhost:888/prometheus + ref: 0.0.1 + optionalComponents: + - upload-image # deploys podinfo as an optional component and apache as a required component -# - name: podinfo-nginx-apache -# path: ../../packages/podinfo-nginx-apache -# ref: 0.0.1 -# optionalComponents: -# - podinfo + - name: podinfo-nginx-apache + path: ../../packages/podinfo-nginx + ref: 0.0.1 + optionalComponents: + - podinfo diff --git a/src/test/e2e/optional_bundle_test.go b/src/test/e2e/optional_bundle_test.go index 4f53ceb1..13a52f2f 100644 --- a/src/test/e2e/optional_bundle_test.go +++ b/src/test/e2e/optional_bundle_test.go @@ -17,12 +17,11 @@ import ( ) func TestBundleOptionalComponents(t *testing.T) { - deployZarfInit(t) e2e.SetupDockerRegistry(t, 888) defer e2e.TeardownRegistry(t, 888) // create 2 Zarf pkgs to be bundled - zarfPkgPath := "src/test/packages/podinfo-and-nginx" + zarfPkgPath := "src/test/packages/podinfo-nginx" e2e.CreateZarfPkg(t, zarfPkgPath, false) zarfPkgPath = "src/test/packages/prometheus" @@ -90,8 +89,8 @@ func introspectOptionalComponentsBundle(t *testing.T) { err = json.Unmarshal(rootManifesBytes, &bundleRootManifest) require.NoError(t, err) - // grab the first pkg (note that it came from a remote source) - pkgManifestBytes, err := os.ReadFile(filepath.Join(blobsDir, bundleRootManifest.Layers[0].Digest.Encoded())) + // grab the second pkg (note that it came from a remote source) + pkgManifestBytes, err := os.ReadFile(filepath.Join(blobsDir, bundleRootManifest.Layers[1].Digest.Encoded())) require.NoError(t, err) remotePkgManifest := ocispec.Manifest{} err = json.Unmarshal(pkgManifestBytes, &remotePkgManifest) @@ -112,8 +111,8 @@ func introspectOptionalComponentsBundle(t *testing.T) { } require.True(t, verifyComponentNotIncluded) - // grab the second pkg (note that it came from a local source) - pkgManifestBytes, err = os.ReadFile(filepath.Join(blobsDir, bundleRootManifest.Layers[1].Digest.Encoded())) + // grab the third pkg (note that it came from a local source) + pkgManifestBytes, err = os.ReadFile(filepath.Join(blobsDir, bundleRootManifest.Layers[2].Digest.Encoded())) require.NoError(t, err) localPkgManifest := ocispec.Manifest{} err = json.Unmarshal(pkgManifestBytes, &localPkgManifest) diff --git a/src/test/packages/podinfo-nginx-apache/apache.yaml b/src/test/packages/podinfo-nginx-apache/apache.yaml deleted file mode 100644 index e69de29b..00000000 diff --git a/src/test/packages/podinfo-nginx/apache.yaml b/src/test/packages/podinfo-nginx/apache.yaml new file mode 100644 index 00000000..4adfa085 --- /dev/null +++ b/src/test/packages/podinfo-nginx/apache.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: apache + labels: + app: httpd_app +spec: + replicas: 1 + selector: + matchLabels: + app: httpd_app + template: + metadata: + labels: + app: httpd_app + spec: + containers: + - name: httpd-container-devops + image: httpd:latest + ports: + - containerPort: 80 diff --git a/src/test/packages/podinfo-nginx-apache/zarf.yaml b/src/test/packages/podinfo-nginx/zarf.yaml similarity index 68% rename from src/test/packages/podinfo-nginx-apache/zarf.yaml rename to src/test/packages/podinfo-nginx/zarf.yaml index 4ab2da3c..b77d4214 100644 --- a/src/test/packages/podinfo-nginx-apache/zarf.yaml +++ b/src/test/packages/podinfo-nginx/zarf.yaml @@ -6,15 +6,15 @@ metadata: components: # only req'd component in this pkg - - name: apache - required: true - manifests: - - name: manifests - namespace: httpbin - files: - - "httpbin.yaml" - images: - - docker.io/kong/httpbin:latest +# - name: apache +# required: true +# manifests: +# - name: manifests +# namespace: apache +# files: +# - "apache.yaml" +# images: +# - httpd:latest - name: podinfo import: diff --git a/tasks/tests.yaml b/tasks/tests.yaml index 606c967b..647c7411 100644 --- a/tasks/tests.yaml +++ b/tasks/tests.yaml @@ -36,3 +36,12 @@ tasks: description: removes bundles and zarf packages that have been created from previous test runs actions: - cmd: find src/test -type f -name '*.tar.zst' -delete + + - name: ci-uds-core-smoke-test + description: deploy UDS Core and runs a couple of validations + actions: + - cmd: build/uds deploy k3d-core-istio-dev:0.16.1 --confirm + - cmd: | + build/uds zarf tools wait-for gateway admin-gateway -n istio-admin-gateway --timeout 10s + build/uds zarf tools wait-for gateway admin-gateway -n istio-admin-gateway --timeout 10s + build/uds zarf tools wait-for gateway admin-gateway -n istio-admin-gateway --timeout 10s From 4fc64cbe997b0153edec28fa35f8256950abcaf3 Mon Sep 17 00:00:00 2001 From: unclegedd Date: Wed, 29 May 2024 10:19:29 -0500 Subject: [PATCH 4/5] removes extra file --- src/test/packages/podinfo-nginx/apache.yaml | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 src/test/packages/podinfo-nginx/apache.yaml diff --git a/src/test/packages/podinfo-nginx/apache.yaml b/src/test/packages/podinfo-nginx/apache.yaml deleted file mode 100644 index 4adfa085..00000000 --- a/src/test/packages/podinfo-nginx/apache.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: apache - labels: - app: httpd_app -spec: - replicas: 1 - selector: - matchLabels: - app: httpd_app - template: - metadata: - labels: - app: httpd_app - spec: - containers: - - name: httpd-container-devops - image: httpd:latest - ports: - - containerPort: 80 From 14fd7526857399b9bf5499f03802ae17260c7f24 Mon Sep 17 00:00:00 2001 From: unclegedd Date: Wed, 29 May 2024 10:20:08 -0500 Subject: [PATCH 5/5] removes comments --- src/test/packages/podinfo-nginx/zarf.yaml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/test/packages/podinfo-nginx/zarf.yaml b/src/test/packages/podinfo-nginx/zarf.yaml index b77d4214..eb0dd34a 100644 --- a/src/test/packages/podinfo-nginx/zarf.yaml +++ b/src/test/packages/podinfo-nginx/zarf.yaml @@ -5,17 +5,6 @@ metadata: version: 0.0.1 components: - # only req'd component in this pkg -# - name: apache -# required: true -# manifests: -# - name: manifests -# namespace: apache -# files: -# - "apache.yaml" -# images: -# - httpd:latest - - name: podinfo import: path: ../podinfo