diff --git a/src/cmd/uds.go b/src/cmd/uds.go index e6630f08..ce266a7e 100644 --- a/src/cmd/uds.go +++ b/src/cmd/uds.go @@ -255,8 +255,7 @@ func configureZarf() { TempDirectory: config.CommonOptions.TempDirectory, OCIConcurrency: config.CommonOptions.OCIConcurrency, Confirm: config.CommonOptions.Confirm, - // todo: decouple Zarf cache? - CachePath: config.CommonOptions.CachePath, + CachePath: config.CommonOptions.CachePath, // use uds-cache instead of zarf-cache } } diff --git a/src/config/config.go b/src/config/config.go index ae9c6101..c122fb20 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -47,6 +47,9 @@ const ( // UDSCache is the directory containing cached bundle layers UDSCache = ".uds-cache" + // UDSCacheLayers is the directory in the cache containing cached bundle layers + UDSCacheLayers = "layers" + // TasksYAML is the default name of the uds run cmd file TasksYAML = "tasks.yaml" diff --git a/src/pkg/bundle/provider.go b/src/pkg/bundle/provider.go index 214de09c..0d5fe3c2 100644 --- a/src/pkg/bundle/provider.go +++ b/src/pkg/bundle/provider.go @@ -34,7 +34,7 @@ type Provider interface { // LoadBundle loads a bundle into the temporary directory and returns a map of the bundle's files // // (currently only the remote provider utilizes the concurrency parameter) - LoadBundle(concurrency int) (types.PathMap, error) + LoadBundle(options types.BundlePullOptions, concurrency int) (*types.UDSBundle, types.PathMap, error) // CreateBundleSBOM creates a bundle-level SBOM from the underlying Zarf packages, if the Zarf package contains an SBOM CreateBundleSBOM(extractSBOM bool) error @@ -43,7 +43,7 @@ type Provider interface { PublishBundle(bundle types.UDSBundle, remote *oci.OrasRemote) error // getBundleManifest gets the bundle's root manifest - getBundleManifest() error + getBundleManifest() (*oci.ZarfOCIManifest, error) // ZarfPackageNameMap returns a map of the zarf package name specified in the uds-bundle.yaml to the actual zarf package name ZarfPackageNameMap() (map[string]string, error) @@ -52,20 +52,34 @@ type Provider interface { // NewBundleProvider returns a new bundler Provider based on the source type func NewBundleProvider(ctx context.Context, source, destination string) (Provider, error) { if helpers.IsOCIURL(source) { - provider := ociProvider{ctx: ctx, src: source, dst: destination} + op := ociProvider{ctx: ctx, src: source, dst: destination} platform := ocispec.Platform{ Architecture: config.GetArch(), OS: oci.MultiOS, } + // get remote client remote, err := oci.NewOrasRemote(source, platform) if err != nil { return nil, err } - provider.OrasRemote = remote - return &provider, nil + op.OrasRemote = remote + + // get root manifest + root, err := op.FetchRoot() + if err != nil { + return nil, err + } + op.rootManifest = root + + return &op, nil } if !utils.IsValidTarballPath(source) { return nil, fmt.Errorf("invalid tarball path: %s", source) } - return &tarballBundleProvider{ctx: ctx, src: source, dst: destination}, nil + tp := tarballBundleProvider{ctx: ctx, src: source, dst: destination} + err := tp.loadBundleManifest() + if err != nil { + return nil, err + } + return &tp, nil } diff --git a/src/pkg/bundle/pull.go b/src/pkg/bundle/pull.go index 695539b6..0322ff04 100644 --- a/src/pkg/bundle/pull.go +++ b/src/pkg/bundle/pull.go @@ -21,10 +21,11 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// Pull pulls a bundle and saves it locally + caches it +// Pull pulls a bundle and saves it locally func (b *Bundle) Pull() error { + // use uds-cache/packages as the dst dir for the pull to get auto caching + // we use an ORAS ocistore to make that dir look like an OCI artifact cacheDir := filepath.Join(zarfConfig.GetAbsCachePath(), "packages") - // create the cache directory if it doesn't exist if err := utils.CreateDirectory(cacheDir, 0o755); err != nil { return err } @@ -41,28 +42,12 @@ func (b *Bundle) Pull() error { return err } - // pull the bundle's metadata + sig - loadedMetadata, err := provider.LoadBundleMetadata() - if err != nil { - return err - } - if err := utils.ReadYaml(loadedMetadata[config.BundleYAML], &b.bundle); err != nil { - return err - } - - // validate the sig (if present) - if err := ValidateBundleSignature(loadedMetadata[config.BundleYAML], loadedMetadata[config.BundleYAMLSignature], b.cfg.PullOpts.PublicKeyPath); err != nil { - return err - } - // pull the bundle's uds-bundle.yaml and it's Zarf pkgs - // todo: refactor this fn, think about pulling the rootDesc first and getting the hashes from there - // today, we are getting the Zarf image manifest hashes from the uds-bundle.yaml - // in that logic we end up pulling the root manifest twice, once in LoadBundle and the other below in remote.ResolveRoot() - loaded, err := provider.LoadBundle(zarfConfig.CommonOptions.OCIConcurrency) + bundle, loaded, err := provider.LoadBundle(b.cfg.PullOpts, zarfConfig.CommonOptions.OCIConcurrency) if err != nil { return err } + b.bundle = *bundle // create a remote client just to resolve the root descriptor platform := ocispec.Platform{ diff --git a/src/pkg/bundle/remote.go b/src/pkg/bundle/remote.go index 74ad523f..0ea38535 100644 --- a/src/pkg/bundle/remote.go +++ b/src/pkg/bundle/remote.go @@ -43,19 +43,14 @@ type ociProvider struct { src string dst string *oci.OrasRemote - manifest *oci.ZarfOCIManifest + rootManifest *oci.ZarfOCIManifest } -func (op *ociProvider) getBundleManifest() error { - if op.manifest != nil { - return nil +func (op *ociProvider) getBundleManifest() (*oci.ZarfOCIManifest, error) { + if op.rootManifest != nil { + return op.rootManifest, nil } - root, err := op.FetchRoot() - if err != nil { - return err - } - op.manifest = root - return nil + return nil, fmt.Errorf("bundle root manifest not loaded") } // LoadBundleMetadata loads a remote bundle's metadata @@ -79,10 +74,6 @@ func (op *ociProvider) LoadBundleMetadata() (types.PathMap, error) { } loaded[rel] = absSha } - err = op.getBundleManifest() - if err != nil { - return nil, err - } return loaded, nil } @@ -149,55 +140,60 @@ func (op *ociProvider) CreateBundleSBOM(extractSBOM bool) error { return nil } -// LoadBundle loads a bundle's uds-bundle.yaml and Zarf packages from a remote source -func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) { - var layersToPull []ocispec.Descriptor - estimatedBytes := int64(0) - - if err := op.getBundleManifest(); err != nil { - return nil, err - } - - loaded, err := op.LoadBundleMetadata() // todo: remove? this seems redundant, can we pass the "loaded" var in +// LoadBundle loads a bundle from a remote source +func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.UDSBundle, types.PathMap, error) { + var bundle types.UDSBundle + // pull the bundle's metadata + sig + loaded, err := op.LoadBundleMetadata() if err != nil { - return nil, err + return nil, nil, err + } + if err := zarfUtils.ReadYaml(loaded[config.BundleYAML], &bundle); err != nil { + return nil, nil, err } - b, err := os.ReadFile(loaded[config.BundleYAML]) - if err != nil { - return nil, err + // validate the sig (if present) before pulling the whole bundle + if err := ValidateBundleSignature(loaded[config.BundleYAML], loaded[config.BundleYAMLSignature], opts.PublicKeyPath); err != nil { + return nil, nil, err } - var bundle types.UDSBundle - if err := goyaml.Unmarshal(b, &bundle); err != nil { - return nil, err + var layersToPull []ocispec.Descriptor + estimatedBytes := int64(0) + + // get the bundle's root manifest + rootManifest, err := op.getBundleManifest() + if err != nil { + return nil, nil, err } for _, pkg := range bundle.Packages { + // grab sha of zarf image manifest and pull it down sha := strings.Split(pkg.Ref, "@sha256:")[1] // this is where we use the SHA appended to the Zarf pkg inside the bundle - manifestDesc := op.manifest.Locate(sha) + manifestDesc := rootManifest.Locate(sha) if err != nil { - return nil, err + return nil, nil, err } manifestBytes, err := op.FetchLayer(manifestDesc) if err != nil { - return nil, err + return nil, nil, err } + // unmarshal the zarf image manifest and add it to the layers to pull var manifest oci.ZarfOCIManifest if err := json.Unmarshal(manifestBytes, &manifest); err != nil { - return nil, err + return nil, nil, err } layersToPull = append(layersToPull, manifestDesc) progressBar := message.NewProgressBar(int64(len(manifest.Layers)), fmt.Sprintf("Verifying layers in Zarf package: %s", pkg.Name)) + // go through the layers in the zarf image manifest and check if they exist in the remote for _, layer := range manifest.Layers { ok, err := op.Repo().Blobs().Exists(op.ctx, layer) progressBar.Add(1) estimatedBytes += layer.Size if err != nil { - return nil, err + return nil, nil, err } // if the layer exists in the remote, add it to the layers to pull if ok { @@ -209,13 +205,13 @@ func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) { store, err := ocistore.NewWithContext(op.ctx, op.dst) if err != nil { - return nil, err + return nil, nil, err } // grab the bundle root manifest and add it to the layers to pull rootDesc, err := op.ResolveRoot() if err != nil { - return nil, err + return nil, nil, err } layersToPull = append(layersToPull, rootDesc) @@ -232,7 +228,7 @@ func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) { _, err = oras.Copy(op.ctx, op.Repo(), op.Repo().Reference.String(), store, op.Repo().Reference.String(), copyOpts) if err != nil { doneSaving <- 1 - return nil, err + return nil, nil, err } doneSaving <- 1 @@ -243,7 +239,7 @@ func (op *ociProvider) LoadBundle(_ int) (types.PathMap, error) { loaded[sha] = filepath.Join(op.dst, config.BlobsDir, sha) } - return loaded, nil + return &bundle, loaded, nil } func (op *ociProvider) PublishBundle(_ types.UDSBundle, _ *oci.OrasRemote) error { @@ -337,7 +333,8 @@ func CheckOCISourcePath(source string) (string, error) { // ZarfPackageNameMap returns the uds bundle zarf package name to actual zarf package name mappings from the oci provider func (op *ociProvider) ZarfPackageNameMap() (map[string]string, error) { - if err := op.getBundleManifest(); err != nil { + rootManifest, err := op.getBundleManifest() + if err != nil { return nil, err } @@ -359,7 +356,7 @@ func (op *ociProvider) ZarfPackageNameMap() (map[string]string, error) { nameMap := make(map[string]string) for _, pkg := range bundle.Packages { sha := strings.Split(pkg.Ref, "@sha256:")[1] // this is where we use the SHA appended to the Zarf pkg inside the bundle - manifestDesc := op.manifest.Locate(sha) + manifestDesc := rootManifest.Locate(sha) nameMap[manifestDesc.Annotations[config.UDSPackageNameAnnotation]] = manifestDesc.Annotations[config.ZarfPackageNameAnnotation] } return nameMap, nil diff --git a/src/pkg/bundle/tarball.go b/src/pkg/bundle/tarball.go index 360b4f41..378be761 100644 --- a/src/pkg/bundle/tarball.go +++ b/src/pkg/bundle/tarball.go @@ -26,16 +26,18 @@ import ( ) type tarballBundleProvider struct { - ctx context.Context - src string - dst string - bundleRootManifest *oci.ZarfOCIManifest - bundleRootDesc ocispec.Descriptor + ctx context.Context + src string + dst string + + // these fields are populated by loadBundleManifest as part of the provider constructor + bundleRootDesc ocispec.Descriptor + rootManifest *oci.ZarfOCIManifest } // CreateBundleSBOM creates a bundle-level SBOM from the underlying Zarf packages, if the Zarf package contains an SBOM func (tp *tarballBundleProvider) CreateBundleSBOM(extractSBOM bool) error { - err := tp.getBundleManifest() + rootManifest, err := tp.getBundleManifest() if err != nil { return err } @@ -47,7 +49,7 @@ func (tp *tarballBundleProvider) CreateBundleSBOM(extractSBOM bool) error { SBOMArtifactPathMap := make(types.PathMap) containsSBOMs := false - for _, layer := range tp.bundleRootManifest.Layers { + for _, layer := range rootManifest.Layers { // get Zarf image manifests from bundle manifest if layer.Annotations[ocispec.AnnotationTitle] == config.BundleYAML { continue @@ -113,10 +115,15 @@ func (tp *tarballBundleProvider) CreateBundleSBOM(extractSBOM bool) error { return nil } -func (tp *tarballBundleProvider) getBundleManifest() error { - if tp.bundleRootManifest != nil { - return nil +func (tp *tarballBundleProvider) getBundleManifest() (*oci.ZarfOCIManifest, error) { + if tp.rootManifest != nil { + return tp.rootManifest, nil } + return nil, fmt.Errorf("bundle root manifest not loaded") +} + +// loadBundleManifest loads the bundle's root manifest and desc into the tarballBundleProvider so we don't have to load it multiple times +func (tp *tarballBundleProvider) loadBundleManifest() error { // Create a secure temporary directory for handling files secureTempDir, err := zarfUtils.MakeTempDir(config.CommonOptions.TempDirectory) if err != nil { @@ -137,7 +144,6 @@ func (tp *tarballBundleProvider) getBundleManifest() error { } var index ocispec.Index - if err := json.Unmarshal(b, &index); err != nil { return fmt.Errorf("failed to unmarshal index.json: %w", err) } @@ -174,18 +180,19 @@ func (tp *tarballBundleProvider) getBundleManifest() error { return err } - tp.bundleRootManifest = manifest + tp.rootManifest = manifest return nil } // LoadBundle loads a bundle from a tarball -func (tp *tarballBundleProvider) LoadBundle(_ int) (types.PathMap, error) { - return nil, fmt.Errorf("uds pull does not support pulling local bundles") +func (tp *tarballBundleProvider) LoadBundle(_ types.BundlePullOptions, _ int) (*types.UDSBundle, types.PathMap, error) { + return nil, nil, fmt.Errorf("uds pull does not support pulling local bundles") } // LoadBundleMetadata loads a bundle's metadata from a tarball func (tp *tarballBundleProvider) LoadBundleMetadata() (types.PathMap, error) { - if err := tp.getBundleManifest(); err != nil { + bundleRootManifest, err := tp.getBundleManifest() + if err != nil { return nil, err } pathsToExtract := config.BundleAlwaysPull @@ -193,7 +200,7 @@ func (tp *tarballBundleProvider) LoadBundleMetadata() (types.PathMap, error) { loaded := make(types.PathMap) for _, path := range pathsToExtract { - layer := tp.bundleRootManifest.Locate(path) + layer := bundleRootManifest.Locate(path) if !oci.IsEmptyDescriptor(layer) { pathInTarball := filepath.Join(config.BlobsDir, layer.Digest.Encoded()) abs := filepath.Join(tp.dst, pathInTarball) @@ -241,7 +248,8 @@ func (tp *tarballBundleProvider) getZarfLayers(store *ocistore.Store, pkgManifes // PublishBundle publishes a local bundle to a remote OCI registry func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *oci.OrasRemote) error { var layersToPush []ocispec.Descriptor - if err := tp.getBundleManifest(); err != nil { + bundleRootManifest, err := tp.getBundleManifest() + if err != nil { return err } estimatedBytes := int64(0) @@ -252,7 +260,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o return err } // push bundle layers to remote - for _, manifestDesc := range tp.bundleRootManifest.Layers { + for _, manifestDesc := range bundleRootManifest.Layers { layersToPush = append(layersToPush, manifestDesc) if manifestDesc.Annotations[ocispec.AnnotationTitle] == config.BundleYAML { continue // uds-bundle.yaml doesn't have layers @@ -266,7 +274,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o } // grab image config - layersToPush = append(layersToPush, tp.bundleRootManifest.Config) + layersToPush = append(layersToPush, bundleRootManifest.Config) // copy bundle copyOpts := utils.CreateCopyOpts(layersToPush, config.CommonOptions.OCIConcurrency) @@ -301,12 +309,13 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o // ZarfPackageNameMap gets zarf package name mappings from tarball provider func (tp *tarballBundleProvider) ZarfPackageNameMap() (map[string]string, error) { - if err := tp.getBundleManifest(); err != nil { + bundleRootManifest, err := tp.getBundleManifest() + if err != nil { return nil, err } nameMap := make(map[string]string) - for _, layer := range tp.bundleRootManifest.Layers { + for _, layer := range bundleRootManifest.Layers { if layer.MediaType == oci.ZarfLayerMediaTypeBlob { // only the uds bundle layer will have AnnotationTitle set if layer.Annotations[ocispec.AnnotationTitle] != config.BundleYAML { diff --git a/src/pkg/cache/cache.go b/src/pkg/cache/cache.go index 78080ae9..88f58e28 100644 --- a/src/pkg/cache/cache.go +++ b/src/pkg/cache/cache.go @@ -30,7 +30,7 @@ func expandTilde(cachePath string) string { func Add(filePathToAdd string) error { // ensure cache dir exists cacheDir := config.CommonOptions.CachePath - if err := os.MkdirAll(filepath.Join(cacheDir, "images"), 0o755); err != nil { + if err := os.MkdirAll(filepath.Join(cacheDir, config.UDSCacheLayers), 0o755); err != nil { return err } @@ -46,7 +46,7 @@ func Add(filePathToAdd string) error { } defer srcFile.Close() - dstFile, err := os.Create(filepath.Join(cacheDir, "images", filename)) + dstFile, err := os.Create(filepath.Join(cacheDir, config.UDSCacheLayers, filename)) if err != nil { return err } @@ -58,7 +58,7 @@ func Add(filePathToAdd string) error { // Exists checks if a layer exists in the cache func Exists(layerDigest string) bool { cacheDir := config.CommonOptions.CachePath - layerCachePath := filepath.Join(expandTilde(cacheDir), "images", layerDigest) + layerCachePath := filepath.Join(expandTilde(cacheDir), config.UDSCacheLayers, layerDigest) _, err := os.Stat(layerCachePath) return !os.IsNotExist(err) } @@ -66,7 +66,7 @@ func Exists(layerDigest string) bool { // Use copies a layer from the cache to the dst dir func Use(layerDigest, dstDir string) error { cacheDir := config.CommonOptions.CachePath - layerCachePath := filepath.Join(expandTilde(cacheDir), "images", layerDigest) + layerCachePath := filepath.Join(expandTilde(cacheDir), config.UDSCacheLayers, layerDigest) srcFile, err := os.Open(layerCachePath) if err != nil { return err diff --git a/src/test/e2e/bundle_test.go b/src/test/e2e/bundle_test.go index cf289fbd..807a2ce7 100644 --- a/src/test/e2e/bundle_test.go +++ b/src/test/e2e/bundle_test.go @@ -21,7 +21,7 @@ import ( ) func zarfPublish(t *testing.T, path string, reg string) { - args := strings.Split(fmt.Sprintf("zarf package publish %s oci://%s --insecure --oci-concurrency=10", path, reg), " ") + args := strings.Split(fmt.Sprintf("zarf package publish %s oci://%s --insecure --oci-concurrency=10 -l debug", path, reg), " ") _, _, err := e2e.UDS(args...) require.NoError(t, err) }