feat: remove unnecessary bundle layers and refactor verification

src/pkg/bundle/deploy.go

@@ -124,7 +124,7 @@ func deployPackages(packages []types.Package, resume bool, b *Bundle) error {
 		// Automatically confirm the package deployment
 		zarfConfig.CommonOptions.Confirm = true
 
-		source, err := sources.New(b.cfg.DeployOpts.Source, pkg.Name, opts, sha, nsOverrides)
+		source, err := sources.New(b.cfg.DeployOpts.Source, pkg, opts, sha, nsOverrides)
 		if err != nil {
 			return err
 		}

src/pkg/bundle/publish.go

@@ -12,14 +12,15 @@ import (
 	"github.com/defenseunicorns/pkg/oci"
 	"github.com/defenseunicorns/uds-cli/src/config"
 	"github.com/defenseunicorns/uds-cli/src/pkg/utils"
+	"github.com/defenseunicorns/uds-cli/src/pkg/utils/boci"
 	"github.com/defenseunicorns/zarf/src/pkg/zoci"
 	av3 "github.com/mholt/archiver/v3"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // Publish publishes a bundle to a remote OCI registry
 func (b *Bundle) Publish() error {
-	b.cfg.PublishOpts.Destination = utils.EnsureOCIPrefix(b.cfg.PublishOpts.Destination)
+	b.cfg.PublishOpts.Destination = boci.EnsureOCIPrefix(b.cfg.PublishOpts.Destination)
 
 	// load bundle metadata into memory
 	// todo: having the tmp dir be the provider.dst is weird

src/pkg/bundle/remote.go

@@ -7,18 +7,17 @@ package bundle
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"slices"
-	"strings"
 
 	"github.com/defenseunicorns/pkg/helpers"
 	"github.com/defenseunicorns/pkg/oci"
 	"github.com/defenseunicorns/uds-cli/src/config"
 	"github.com/defenseunicorns/uds-cli/src/pkg/utils"
+	"github.com/defenseunicorns/uds-cli/src/pkg/utils/boci"
 	"github.com/defenseunicorns/uds-cli/src/types"
 	"github.com/defenseunicorns/zarf/src/pkg/cluster"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
@@ -173,37 +172,13 @@ func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.U
 	layersToPull = append(layersToPull, rootManifest.Config)
 
 	for _, pkg := range bundle.Packages {
-
-		// grab sha of zarf image manifest and pull it down
-		sha := strings.Split(pkg.Ref, "@sha256:")[1] // this is where we use the SHA appended to the Zarf pkg inside the bundle
-		manifestDesc := rootManifest.Locate(sha)
-		manifestBytes, err := op.FetchLayer(ctx, manifestDesc)
+		// go through the pkg's layers and figure out which ones to pull based on the req'd + selected components
+		pkgLayers, estPkgBytes, err := boci.FindBundledPkgLayers(ctx, pkg, rootManifest, op.OrasRemote)
 		if err != nil {
 			return nil, nil, err
 		}
-
-		// unmarshal the zarf image manifest and add it to the layers to pull
-		var manifest oci.Manifest
-		if err := json.Unmarshal(manifestBytes, &manifest); err != nil {
-			return nil, nil, err
-		}
-		layersToPull = append(layersToPull, manifestDesc)
-		progressBar := message.NewProgressBar(int64(len(manifest.Layers)), fmt.Sprintf("Verifying layers in Zarf package: %s", pkg.Name))
-
-		// go through the layers in the zarf image manifest and check if they exist in the remote
-		for _, layer := range manifest.Layers {
-			ok, err := op.Repo().Blobs().Exists(ctx, layer)
-			progressBar.Add(1)
-			estimatedBytes += layer.Size
-			if err != nil {
-				return nil, nil, err
-			}
-			// if the layer exists in the remote, add it to the layers to pull
-			if ok {
-				layersToPull = append(layersToPull, layer)
-			}
-		}
-		progressBar.Successf("Verified %s package", pkg.Name)
+		layersToPull = append(layersToPull, pkgLayers...)
+		estimatedBytes += estPkgBytes
 	}
 
 	store, err := ocistore.NewWithContext(ctx, op.dst)
@@ -219,7 +194,7 @@ func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.U
 	layersToPull = append(layersToPull, rootDesc)
 
 	// create copy options for oras.Copy()
-	copyOpts := utils.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency)
+	copyOpts := boci.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency)
 
 	// Create a thread to update a progress bar as we save the package to disk
 	doneSaving := make(chan error)
@@ -255,7 +230,7 @@ func getOCIValidatedSource(source string) (string, error) {
 		OS:           oci.MultiOS,
 	}
 	// Check provided repository path
-	sourceWithOCI := utils.EnsureOCIPrefix(source)
+	sourceWithOCI := boci.EnsureOCIPrefix(source)
 	remote, err := zoci.NewRemote(sourceWithOCI, platform)
 	if err == nil {
 		source = sourceWithOCI

src/pkg/bundle/remove.go

@@ -93,7 +93,7 @@ func removePackages(packagesToRemove []types.Package, b *Bundle) error {
 			}
 
 			sha := strings.Split(pkg.Ref, "sha256:")[1]
-			source, err := sources.New(b.cfg.RemoveOpts.Source, pkg.Name, opts, sha, nil)
+			source, err := sources.New(b.cfg.RemoveOpts.Source, pkg, opts, sha, nil)
 			if err != nil {
 				return err
 			}

src/pkg/bundle/tarball.go

@@ -16,6 +16,7 @@ import (
 	"github.com/defenseunicorns/pkg/oci"
 	"github.com/defenseunicorns/uds-cli/src/config"
 	"github.com/defenseunicorns/uds-cli/src/pkg/utils"
+	"github.com/defenseunicorns/uds-cli/src/pkg/utils/boci"
 	"github.com/defenseunicorns/uds-cli/src/types"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
 	zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils"
@@ -217,6 +218,7 @@ func (tp *tarballBundleProvider) LoadBundleMetadata() (types.PathMap, error) {
 	return loaded, nil
 }
 
+// getZarfLayers returns the layers of the Zarf package that are in the bundle
 func (tp *tarballBundleProvider) getZarfLayers(store *ocistore.Store, pkgManifestDesc ocispec.Descriptor) ([]ocispec.Descriptor, int64, error) {
 	var layersToPull []ocispec.Descriptor
 	estimatedPkgSize := int64(0)
@@ -278,7 +280,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o
 	layersToPush = append(layersToPush, bundleRootManifest.Config)
 
 	// copy bundle
-	copyOpts := utils.CreateCopyOpts(layersToPush, config.CommonOptions.OCIConcurrency)
+	copyOpts := boci.CreateCopyOpts(layersToPush, config.CommonOptions.OCIConcurrency)
 	progressBar := message.NewProgressBar(estimatedBytes, fmt.Sprintf("Publishing %s:%s", remote.Repo().Reference.Repository, remote.Repo().Reference.Reference))
 	defer progressBar.Stop()
 	remote.SetProgressWriter(progressBar)
@@ -287,7 +289,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o
 	ref := bundle.Metadata.Version
 
 	// check for existing index
-	index, err := utils.GetIndex(remote, ref)
+	index, err := boci.GetIndex(remote, ref)
 	if err != nil {
 		return err
 	}
@@ -315,7 +317,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o
 	}
 
 	// create or update, then push index.json
-	err = utils.UpdateIndex(index, remote, &bundle, tp.bundleRootDesc)
+	err = boci.UpdateIndex(index, remote, &bundle, tp.bundleRootDesc)
 	if err != nil {
 		return err
 	}

src/pkg/bundler/common.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/defenseunicorns/pkg/helpers"
 	"github.com/defenseunicorns/pkg/oci"
-	"github.com/defenseunicorns/uds-cli/src/pkg/utils"
+	"github.com/defenseunicorns/uds-cli/src/pkg/utils/boci"
 	"github.com/defenseunicorns/uds-cli/src/types"
 	"github.com/defenseunicorns/zarf/src/pkg/message"
 	"github.com/defenseunicorns/zarf/src/pkg/zoci"
@@ -55,7 +55,7 @@ func pushManifestConfigFromMetadata(r *oci.OrasRemote, metadata *types.UDSMetada
 		OCIVersion:   "1.0.1",
 		Annotations:  annotations,
 	}
-	manifestConfigDesc, err := utils.ToOCIRemote(manifestConfig, zoci.ZarfLayerMediaTypeBlob, r)
+	manifestConfigDesc, err := boci.ToOCIRemote(manifestConfig, zoci.ZarfLayerMediaTypeBlob, r)
 	if err != nil {
 		return ocispec.Descriptor{}, err
 	}

src/pkg/bundler/fetcher/common.go

@@ -0,0 +1,100 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: 2023-Present The UDS Authors
+
+// Package fetcher contains functionality to fetch local and remote Zarf pkgs for local bundling
+package fetcher
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/defenseunicorns/pkg/helpers"
+	"github.com/defenseunicorns/uds-cli/src/config"
+	"github.com/defenseunicorns/uds-cli/src/pkg/utils"
+	"github.com/defenseunicorns/zarf/src/pkg/layout"
+	"github.com/defenseunicorns/zarf/src/pkg/packager/filters"
+	zarfSources "github.com/defenseunicorns/zarf/src/pkg/packager/sources"
+	zarfTypes "github.com/defenseunicorns/zarf/src/types"
+	goyaml "github.com/goccy/go-yaml"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// loadPkg loads a package from a tarball source and filters out optional components
+func loadPkg(pkgTmp string, pkgSrc zarfSources.PackageSource, optionalComponents []string) (zarfTypes.ZarfPackage, *layout.PackagePaths, error) {
+	// create empty layout and source
+	pkgPaths := layout.New(pkgTmp)
+
+	// create filter for optional components
+	createFilter := filters.Combine(
+		filters.ForDeploy(strings.Join(optionalComponents, ","), false),
+	)
+
+	// load the package with the filter (calling LoadPackage populates the pkgPaths with the files from the tarball)
+	pkg, _, err := pkgSrc.LoadPackage(pkgPaths, createFilter, false)
+	if err != nil {
+		return zarfTypes.ZarfPackage{}, nil, err
+	}
+	return pkg, pkgPaths, nil
+}
+
+// getImgLayerDigests grabs the digests of the layers from the images in the image index
+func getImgLayerDigests(manifestsToInclude []ocispec.Descriptor, pkgPaths *layout.PackagePaths) ([]string, error) {
+	var includeLayers []string
+	for _, manifest := range manifestsToInclude {
+		includeLayers = append(includeLayers, manifest.Digest.Hex()) // be sure to include image manifest
+		manifestBytes, err := os.ReadFile(filepath.Join(pkgPaths.Images.Base, config.BlobsDir, manifest.Digest.Hex()))
+		if err != nil {
+			return nil, err
+		}
+		var imgManifest ocispec.Manifest
+		err = goyaml.Unmarshal(manifestBytes, &imgManifest)
+		if err != nil {
+			return nil, err
+		}
+		includeLayers = append(includeLayers, imgManifest.Config.Digest.Hex()) // don't forget the config
+		for _, layer := range imgManifest.Layers {
+			includeLayers = append(includeLayers, layer.Digest.Hex())
+		}
+	}
+	return includeLayers, nil
+}
+
+// filterPkgPaths grabs paths that either not in the blobs dir or are in includeLayers
+func filterPkgPaths(pkgPaths *layout.PackagePaths, includeLayers []string, optionalComponents []zarfTypes.ZarfComponent) []string {
+	var filteredPaths []string
+	paths := pkgPaths.Files()
+	for _, path := range paths {
+		// include all paths that aren't in the blobs dir
+		if !strings.Contains(path, config.BlobsDir) {
+			// only grab req'd + specified optional components
+			if strings.Contains(path, "/components/") {
+				if shouldInclude := utils.IncludeComponent(path, optionalComponents); shouldInclude {
+					filteredPaths = append(filteredPaths, path)
+					continue
+				}
+			} else {
+				filteredPaths = append(filteredPaths, path)
+			}
+		}
+		// include paths that are in the blobs dir and are in includeLayers
+		for _, layer := range includeLayers {
+			if strings.Contains(path, config.BlobsDir) && strings.Contains(path, layer) {
+				filteredPaths = append(filteredPaths, path)
+				break
+			}
+		}
+	}
+
+	// ensure zarf.yaml, checksums and SBOMS (if exists) are always included
+	// note we may have extra SBOMs because they are not filtered or modified
+	alwaysInclude := []string{pkgPaths.ZarfYAML, pkgPaths.Checksums}
+	if pkgPaths.SBOMs.Path != "" {
+		alwaysInclude = append(alwaysInclude, pkgPaths.SBOMs.Path)
+	}
+	filteredPaths = helpers.MergeSlices(filteredPaths, alwaysInclude, func(a, b string) bool {
+		return a == b
+	})
+
+	return filteredPaths
+}