From fae1a19f133e09d4684290c3015f428d9c15792c Mon Sep 17 00:00:00 2001 From: Darcy Cleaver Date: Mon, 11 Mar 2024 09:37:23 -0600 Subject: [PATCH 1/7] compiles --- go.mod | 8 +++-- go.sum | 15 ++++---- src/config/config.go | 5 ++- src/pkg/bundle/common.go | 8 +++-- src/pkg/bundle/deploy.go | 5 +-- src/pkg/bundle/inspect.go | 2 +- src/pkg/bundle/provider.go | 9 ++--- src/pkg/bundle/publish.go | 5 +-- src/pkg/bundle/pull.go | 13 ++++--- src/pkg/bundle/remote.go | 55 ++++++++++++++---------------- src/pkg/bundle/remove.go | 2 +- src/pkg/bundle/tarball.go | 21 +++++++----- src/pkg/bundler/common.go | 8 +++-- src/pkg/bundler/fetcher/fetcher.go | 8 +++-- src/pkg/bundler/fetcher/local.go | 7 ++-- src/pkg/bundler/fetcher/remote.go | 30 +++++++--------- src/pkg/bundler/localbundle.go | 7 ++-- src/pkg/bundler/pusher/remote.go | 29 ++++++---------- src/pkg/bundler/remotebundle.go | 30 +++++++++------- src/pkg/sources/new.go | 7 ++-- src/pkg/sources/remote.go | 29 +++++++--------- src/pkg/sources/tarball.go | 4 +-- src/pkg/utils/oci.go | 34 +++++++++--------- 23 files changed, 173 insertions(+), 168 deletions(-) diff --git a/go.mod b/go.mod index a22f1270..dc575e7f 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,10 @@ module github.com/defenseunicorns/uds-cli go 1.21.6 +replace github.com/defenseunicorns/zarf => /Users/darcy/leapfrog/gh-repos/defenseunicorns/zarf + +replace github.com/defenseunicorns/maru-runner => /Users/darcy/leapfrog/gh-repos/defenseunicorns/maru-runner + require ( github.com/AlecAivazis/survey/v2 v2.3.7 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b @@ -15,7 +19,7 @@ require ( github.com/pterm/pterm v0.12.79 github.com/spf13/cobra v1.8.0 github.com/spf13/viper v1.18.2 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 golang.org/x/mod v0.16.0 golang.org/x/sync v0.6.0 @@ -86,7 +90,7 @@ require ( github.com/alibabacloud-go/tea-xml v1.1.3 // indirect github.com/aliyun/credentials-go v1.3.1 // indirect github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 // indirect - github.com/anchore/clio v0.0.0-20240209204744-cb94e40a4f65 // indirect + github.com/anchore/clio v0.0.0-20240301210832-abcb7197da84 // indirect github.com/anchore/fangs v0.0.0-20231201140849-5075d28d6d8b // indirect github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a // indirect github.com/anchore/go-macholibre v0.0.0-20220308212642-53e6d0aaf6fb // indirect diff --git a/go.sum b/go.sum index e944ae07..60ddddad 100644 --- a/go.sum +++ b/go.sum @@ -364,8 +364,8 @@ github.com/aliyun/credentials-go v1.3.1 h1:uq/0v7kWrxmoLGpqjx7vtQ/s03f0zR//0br/x github.com/aliyun/credentials-go v1.3.1/go.mod h1:8jKYhQuDawt8x2+fusqa1Y6mPxemTsBEN04dgcAcYz0= github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9 h1:p0ZIe0htYOX284Y4axJaGBvXHU0VCCzLN5Wf5XbKStU= github.com/anchore/bubbly v0.0.0-20231115134915-def0aba654a9/go.mod h1:3ZsFB9tzW3vl4gEiUeuSOMDnwroWxIxJelOOHUp8dSw= -github.com/anchore/clio v0.0.0-20240209204744-cb94e40a4f65 h1:u9XrEabKlGPsrmRvAER+kUKkwXiJfLyqGhmOTFsXjX4= -github.com/anchore/clio v0.0.0-20240209204744-cb94e40a4f65/go.mod h1:8Jr7CjmwFVcBPtkJdTpaAGHimoGJGfbExypjzOu87Og= +github.com/anchore/clio v0.0.0-20240301210832-abcb7197da84 h1:z8C0o7URkxvuh6isbZbz3la5m6sINzIOmt6UfZ4ey1o= +github.com/anchore/clio v0.0.0-20240301210832-abcb7197da84/go.mod h1:XEh8WpqQSsNNZZjCZ6xWAk/DoNYfyISjUQ+Bv06xHEc= github.com/anchore/fangs v0.0.0-20231201140849-5075d28d6d8b h1:L/djgY7ZbZ/38+wUtdkk398W3PIBJLkt1N8nU/7e47A= github.com/anchore/fangs v0.0.0-20231201140849-5075d28d6d8b/go.mod h1:TLcE0RE5+8oIx2/NPWem/dq1DeaMoC+fPEH7hoSzPLo= github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a h1:nJ2G8zWKASyVClGVgG7sfM5mwoZlZ2zYpIzN2OhjWkw= @@ -598,10 +598,6 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX8ATG8oKsE= github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c= -github.com/defenseunicorns/maru-runner v0.0.1 h1:0SPiaXbPKnv7bjsUW2f7rPiIMmd3YLfT9+wQe4810K8= -github.com/defenseunicorns/maru-runner v0.0.1/go.mod h1:3K+JeLpud+rb8vC+nPFaTNjhqW40++6qFKKVTBEEzQM= -github.com/defenseunicorns/zarf v0.32.4 h1:3foCaUHUtAu8YId49j3u+EVknaTB8ERaQ9J6Do+bAwc= -github.com/defenseunicorns/zarf v0.32.4/go.mod h1:f4H7al7qnj5VXfkUkB/CcepVW/DA/O5tvAy8TWv9aT8= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da/go.mod h1:B3tI9iGHi4imdLi4Asdha1Sc6feLMTfPLXh9IUYmysk= github.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936 h1:foGzavPWwtoyBvjWyKJYDYsyzy+23iBV7NKTwdk+LRY= @@ -1588,8 +1584,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.1 h1:4VhoImhV/Bm0ToFkXFi8hXNXwpDRZ/ynw3amt82mzq0= -github.com/stretchr/objx v0.5.1/go.mod h1:/iHQpkQwBD6DLUmQ4pE+s1TXdob1mORJ4/UFdrifcy0= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1602,8 +1598,9 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= diff --git a/src/config/config.go b/src/config/config.go index c122fb20..c6658218 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -9,7 +9,6 @@ import ( "time" "github.com/defenseunicorns/uds-cli/src/types" - zarfConfig "github.com/defenseunicorns/zarf/src/config" zarfTypes "github.com/defenseunicorns/zarf/src/types" ) @@ -114,9 +113,9 @@ var ( // could technically remove, but it doesn't hurt anything for now var DefaultZarfInitOptions = zarfTypes.ZarfInitOptions{ GitServer: zarfTypes.GitServerInfo{ - PushUsername: zarfConfig.ZarfGitPushUser, + PushUsername: zarfTypes.ZarfGitPushUser, }, RegistryInfo: zarfTypes.RegistryInfo{ - PushUsername: zarfConfig.ZarfRegistryPushUser, + PushUsername: zarfTypes.ZarfRegistryPushUser, }, } diff --git a/src/pkg/bundle/common.go b/src/pkg/bundle/common.go index 86276941..54c34284 100644 --- a/src/pkg/bundle/common.go +++ b/src/pkg/bundle/common.go @@ -5,6 +5,7 @@ package bundle import ( + "context" "errors" "fmt" "os" @@ -21,6 +22,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" zarfTypes "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -139,12 +141,12 @@ func (b *Bundle) ValidateBundleResources(bundle *types.UDSBundle, spinner *messa Architecture: config.GetArch(), OS: oci.MultiOS, } - remote, err := oci.NewOrasRemote(url, platform) + remote, err := zoci.NewRemote(url, platform) if err != nil { return err } if err := remote.Repo().Reference.ValidateReferenceAsDigest(); err != nil { - manifestDesc, err := remote.ResolveRoot() + manifestDesc, err := remote.ResolveRoot(context.TODO()) if err != nil { return err } @@ -186,7 +188,7 @@ func (b *Bundle) ValidateBundleResources(bundle *types.UDSBundle, spinner *messa // todo: need to packager.ValidatePackageSignature (or come up with a bundle-level signature scheme) publicKeyPath := filepath.Join(b.tmp, config.PublicKeyFile) if pkg.PublicKey != "" { - if err := utils.WriteFile(publicKeyPath, []byte(pkg.PublicKey)); err != nil { + if err := os.WriteFile(publicKeyPath, []byte(pkg.PublicKey), helpers.ReadWriteUser); err != nil { return err } defer os.Remove(publicKeyPath) diff --git a/src/pkg/bundle/deploy.go b/src/pkg/bundle/deploy.go index cdb834fd..9792d8eb 100644 --- a/src/pkg/bundle/deploy.go +++ b/src/pkg/bundle/deploy.go @@ -21,6 +21,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/packager" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" zarfTypes "github.com/defenseunicorns/zarf/src/types" "github.com/pterm/pterm" "golang.org/x/exp/slices" @@ -44,7 +45,7 @@ func (b *Bundle) Deploy() error { defer metadataSpinner.Stop() // Check that provided oci source path is valid, and update it if it's missing the full path - source, err := CheckOCISourcePath(b.cfg.DeployOpts.Source) + source, err := CheckOCISourcePath(ctx, b.cfg.DeployOpts.Source) if err != nil { return err } @@ -144,7 +145,7 @@ func deployPackages(packages []types.Package, resume bool, b *Bundle, zarfPackag publicKeyPath := filepath.Join(b.tmp, config.PublicKeyFile) if pkg.PublicKey != "" { - if err := utils.WriteFile(publicKeyPath, []byte(pkg.PublicKey)); err != nil { + if err := os.WriteFile(publicKeyPath, []byte(pkg.PublicKey), helpers.ReadWriteUser); err != nil { return err } defer os.Remove(publicKeyPath) diff --git a/src/pkg/bundle/inspect.go b/src/pkg/bundle/inspect.go index 9121bdbd..f88eddcf 100644 --- a/src/pkg/bundle/inspect.go +++ b/src/pkg/bundle/inspect.go @@ -16,7 +16,7 @@ func (b *Bundle) Inspect() error { ctx := context.TODO() // Check that provided oci source path is valid, and update it if it's missing the full path - source, err := CheckOCISourcePath(b.cfg.InspectOpts.Source) + source, err := CheckOCISourcePath(ctx, b.cfg.InspectOpts.Source) if err != nil { return err } diff --git a/src/pkg/bundle/provider.go b/src/pkg/bundle/provider.go index 0d5fe3c2..50996e2d 100644 --- a/src/pkg/bundle/provider.go +++ b/src/pkg/bundle/provider.go @@ -13,6 +13,7 @@ import ( "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -43,7 +44,7 @@ type Provider interface { PublishBundle(bundle types.UDSBundle, remote *oci.OrasRemote) error // getBundleManifest gets the bundle's root manifest - getBundleManifest() (*oci.ZarfOCIManifest, error) + getBundleManifest() (*oci.Manifest, error) // ZarfPackageNameMap returns a map of the zarf package name specified in the uds-bundle.yaml to the actual zarf package name ZarfPackageNameMap() (map[string]string, error) @@ -58,14 +59,14 @@ func NewBundleProvider(ctx context.Context, source, destination string) (Provide OS: oci.MultiOS, } // get remote client - remote, err := oci.NewOrasRemote(source, platform) + remote, err := zoci.NewRemote(source, platform) if err != nil { return nil, err } - op.OrasRemote = remote + op.OrasRemote = remote.OrasRemote // get root manifest - root, err := op.FetchRoot() + root, err := op.FetchRoot(ctx) if err != nil { return nil, err } diff --git a/src/pkg/bundle/publish.go b/src/pkg/bundle/publish.go index 6ac886eb..f4921613 100644 --- a/src/pkg/bundle/publish.go +++ b/src/pkg/bundle/publish.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/uds-cli/src/pkg/utils" "github.com/defenseunicorns/zarf/src/pkg/oci" zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/zoci" av3 "github.com/mholt/archiver/v3" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -54,11 +55,11 @@ func (b *Bundle) Publish() error { Architecture: config.GetArch(), OS: oci.MultiOS, } - remote, err := oci.NewOrasRemote(fmt.Sprintf("%s/%s:%s", ociURL, bundleName, bundleTag), platform) + remote, err := zoci.NewRemote(fmt.Sprintf("%s/%s:%s", ociURL, bundleName, bundleTag), platform) if err != nil { return err } - err = provider.PublishBundle(b.bundle, remote) + err = provider.PublishBundle(b.bundle, remote.OrasRemote) if err != nil { return err } diff --git a/src/pkg/bundle/pull.go b/src/pkg/bundle/pull.go index 0322ff04..3d6d8cfe 100644 --- a/src/pkg/bundle/pull.go +++ b/src/pkg/bundle/pull.go @@ -17,12 +17,15 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" "github.com/mholt/archiver/v4" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // Pull pulls a bundle and saves it locally func (b *Bundle) Pull() error { + ctx := context.TODO() // use uds-cache/packages as the dst dir for the pull to get auto caching // we use an ORAS ocistore to make that dir look like an OCI artifact cacheDir := filepath.Join(zarfConfig.GetAbsCachePath(), "packages") @@ -31,13 +34,13 @@ func (b *Bundle) Pull() error { } // Get validated source path - source, err := CheckOCISourcePath(b.cfg.PullOpts.Source) + source, err := CheckOCISourcePath(ctx, b.cfg.PullOpts.Source) if err != nil { return err } b.cfg.PullOpts.Source = source - provider, err := NewBundleProvider(context.TODO(), b.cfg.PullOpts.Source, cacheDir) + provider, err := NewBundleProvider(ctx, b.cfg.PullOpts.Source, cacheDir) if err != nil { return err } @@ -54,13 +57,13 @@ func (b *Bundle) Pull() error { Architecture: config.GetArch(), OS: oci.MultiOS, } - remote, err := oci.NewOrasRemote(b.cfg.PullOpts.Source, platform) + remote, err := zoci.NewRemote(b.cfg.PullOpts.Source, platform) if err != nil { return err } // fetch the bundle's root descriptor - rootDesc, err := remote.ResolveRoot() + rootDesc, err := remote.ResolveRoot(ctx) if err != nil { return err } @@ -79,7 +82,7 @@ func (b *Bundle) Pull() error { return err } indexJSONPath := filepath.Join(b.tmp, "index.json") - if err := utils.WriteFile(indexJSONPath, bytes); err != nil { + if err := os.WriteFile(indexJSONPath, bytes, helpers.ReadWriteUser); err != nil { return err } diff --git a/src/pkg/bundle/remote.go b/src/pkg/bundle/remote.go index 0ea38535..6815bb2c 100644 --- a/src/pkg/bundle/remote.go +++ b/src/pkg/bundle/remote.go @@ -13,7 +13,6 @@ import ( "path/filepath" "slices" "strings" - "sync" "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/uds-cli/src/pkg/utils" @@ -22,6 +21,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/zoci" goyaml "github.com/goccy/go-yaml" "github.com/mholt/archiver/v4" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -43,10 +43,10 @@ type ociProvider struct { src string dst string *oci.OrasRemote - rootManifest *oci.ZarfOCIManifest + rootManifest *oci.Manifest } -func (op *ociProvider) getBundleManifest() (*oci.ZarfOCIManifest, error) { +func (op *ociProvider) getBundleManifest() (*oci.Manifest, error) { if op.rootManifest != nil { return op.rootManifest, nil } @@ -59,7 +59,7 @@ func (op *ociProvider) LoadBundleMetadata() (types.PathMap, error) { return nil, err } - layers, err := op.PullPackagePaths(config.BundleAlwaysPull, filepath.Join(op.dst, config.BlobsDir)) + layers, err := op.PullPaths(op.ctx, filepath.Join(op.dst, config.BlobsDir), config.BundleAlwaysPull) if err != nil { return nil, err } @@ -80,7 +80,7 @@ func (op *ociProvider) LoadBundleMetadata() (types.PathMap, error) { // CreateBundleSBOM creates a bundle-level SBOM from the underlying Zarf packages, if the Zarf package contains an SBOM func (op *ociProvider) CreateBundleSBOM(extractSBOM bool) error { SBOMArtifactPathMap := make(types.PathMap) - root, err := op.FetchRoot() + root, err := op.FetchRoot(op.ctx) if err != nil { return err } @@ -96,7 +96,7 @@ func (op *ociProvider) CreateBundleSBOM(extractSBOM bool) error { if layer.Annotations[ocispec.AnnotationTitle] == config.BundleYAML { continue } - zarfManifest, err := op.OrasRemote.FetchManifest(layer) + zarfManifest, err := op.OrasRemote.FetchManifest(op.ctx, layer) if err != nil { return err } @@ -107,7 +107,7 @@ func (op *ociProvider) CreateBundleSBOM(extractSBOM bool) error { continue } // grab sboms.tar and extract - sbomBytes, err := op.OrasRemote.FetchLayer(sbomDesc) + sbomBytes, err := op.OrasRemote.FetchLayer(op.ctx, sbomDesc) if err != nil { return err } @@ -174,13 +174,13 @@ func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.U if err != nil { return nil, nil, err } - manifestBytes, err := op.FetchLayer(manifestDesc) + manifestBytes, err := op.FetchLayer(op.ctx, manifestDesc) if err != nil { return nil, nil, err } // unmarshal the zarf image manifest and add it to the layers to pull - var manifest oci.ZarfOCIManifest + var manifest oci.Manifest if err := json.Unmarshal(manifestBytes, &manifest); err != nil { return nil, nil, err } @@ -209,7 +209,7 @@ func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.U } // grab the bundle root manifest and add it to the layers to pull - rootDesc, err := op.ResolveRoot() + rootDesc, err := op.ResolveRoot(op.ctx) if err != nil { return nil, nil, err } @@ -219,21 +219,16 @@ func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.U copyOpts := utils.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency) // Create a thread to update a progress bar as we save the package to disk - doneSaving := make(chan int) - errChan := make(chan int) - var wg sync.WaitGroup - wg.Add(1) - go zarfUtils.RenderProgressBarForLocalDirWrite(op.dst, estimatedBytes, &wg, doneSaving, errChan, fmt.Sprintf("Pulling bundle: %s", bundle.Metadata.Name), fmt.Sprintf("Successfully pulled bundle: %s", bundle.Metadata.Name)) + doneSaving := make(chan error) + go zarfUtils.RenderProgressBarForLocalDirWrite(op.dst, estimatedBytes, doneSaving, fmt.Sprintf("Pulling bundle: %s", bundle.Metadata.Name), fmt.Sprintf("Successfully pulled bundle: %s", bundle.Metadata.Name)) // note that in this case oras.Copy() copies using the bundle root manifest, not the packages directly _, err = oras.Copy(op.ctx, op.Repo(), op.Repo().Reference.String(), store, op.Repo().Reference.String(), copyOpts) + doneSaving <- err + <-doneSaving if err != nil { - doneSaving <- 1 return nil, nil, err } - doneSaving <- 1 - wg.Wait() - for _, layer := range layersToPull { sha := layer.Digest.Encoded() loaded[sha] = filepath.Join(op.dst, config.BlobsDir, sha) @@ -248,7 +243,7 @@ func (op *ociProvider) PublishBundle(_ types.UDSBundle, _ *oci.OrasRemote) error } // Returns the validated source path based on the provided oci source path -func getOCIValidatedSource(source string) (string, error) { +func getOCIValidatedSource(ctx context.Context, source string) (string, error) { originalSource := source platform := ocispec.Platform{ @@ -257,35 +252,35 @@ func getOCIValidatedSource(source string) (string, error) { } // Check provided repository path sourceWithOCI := utils.EnsureOCIPrefix(source) - remote, err := oci.NewOrasRemote(sourceWithOCI, platform) + remote, err := zoci.NewRemote(sourceWithOCI, platform) if err == nil { source = sourceWithOCI - _, err = remote.ResolveRoot() + _, err = remote.ResolveRoot(ctx) } // if root didn't resolve, expand the path if err != nil { // Check in ghcr uds bundle path source = GHCRUDSBundlePath + originalSource - remote, err = oci.NewOrasRemote(source, platform) + remote, err = zoci.NewRemote(source, platform) if err == nil { - _, err = remote.ResolveRoot() + _, err = remote.ResolveRoot(ctx) } if err != nil { message.Debugf("%s: not found", source) // Check in delivery bundle path source = GHCRDeliveryBundlePath + originalSource - remote, err = oci.NewOrasRemote(source, platform) + remote, err = zoci.NewRemote(source, platform) if err == nil { - _, err = remote.ResolveRoot() + _, err = remote.ResolveRoot(ctx) } if err != nil { message.Debugf("%s: not found", source) // Check in packages bundle path source = GHCRPackagesPath + originalSource - remote, err = oci.NewOrasRemote(source, platform) + remote, err = zoci.NewRemote(source, platform) if err == nil { - _, err = remote.ResolveRoot() + _, err = remote.ResolveRoot(ctx) } if err != nil { message.Fatalf(nil, "%s: not found", originalSource) @@ -319,11 +314,11 @@ func ValidateArch(arch string) error { } // CheckOCISourcePath checks that provided oci source path is valid, and updates it if it's missing the full path -func CheckOCISourcePath(source string) (string, error) { +func CheckOCISourcePath(ctx context.Context, source string) (string, error) { validTarballPath := utils.IsValidTarballPath(source) var err error if !validTarballPath { - source, err = getOCIValidatedSource(source) + source, err = getOCIValidatedSource(ctx, source) if err != nil { return "", err } diff --git a/src/pkg/bundle/remove.go b/src/pkg/bundle/remove.go index 6896bfcc..caf53cbc 100644 --- a/src/pkg/bundle/remove.go +++ b/src/pkg/bundle/remove.go @@ -25,7 +25,7 @@ func (b *Bundle) Remove() error { ctx := context.TODO() // Check that provided oci source path is valid, and update it if it's missing the full path - source, err := CheckOCISourcePath(b.cfg.RemoveOpts.Source) + source, err := CheckOCISourcePath(ctx, b.cfg.RemoveOpts.Source) if err != nil { return err } diff --git a/src/pkg/bundle/tarball.go b/src/pkg/bundle/tarball.go index 378be761..52f1116b 100644 --- a/src/pkg/bundle/tarball.go +++ b/src/pkg/bundle/tarball.go @@ -18,6 +18,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/zoci" av3 "github.com/mholt/archiver/v3" av4 "github.com/mholt/archiver/v4" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -32,7 +33,7 @@ type tarballBundleProvider struct { // these fields are populated by loadBundleManifest as part of the provider constructor bundleRootDesc ocispec.Descriptor - rootManifest *oci.ZarfOCIManifest + rootManifest *oci.Manifest } // CreateBundleSBOM creates a bundle-level SBOM from the underlying Zarf packages, if the Zarf package contains an SBOM @@ -64,7 +65,7 @@ func (tp *tarballBundleProvider) CreateBundleSBOM(extractSBOM bool) error { if err != nil { return err } - var zarfImageManifest *oci.ZarfOCIManifest + var zarfImageManifest *oci.Manifest if err := json.Unmarshal(zarfManifestBytes, &zarfImageManifest); err != nil { return err } @@ -115,7 +116,7 @@ func (tp *tarballBundleProvider) CreateBundleSBOM(extractSBOM bool) error { return nil } -func (tp *tarballBundleProvider) getBundleManifest() (*oci.ZarfOCIManifest, error) { +func (tp *tarballBundleProvider) getBundleManifest() (*oci.Manifest, error) { if tp.rootManifest != nil { return tp.rootManifest, nil } @@ -174,7 +175,7 @@ func (tp *tarballBundleProvider) loadBundleManifest() error { return err } - var manifest *oci.ZarfOCIManifest + var manifest *oci.Manifest if err := json.Unmarshal(b, &manifest); err != nil { return err @@ -225,7 +226,7 @@ func (tp *tarballBundleProvider) getZarfLayers(store *ocistore.Store, pkgManifes return nil, int64(0), err } - var zarfImageManifest *oci.ZarfOCIManifest + var zarfImageManifest *oci.Manifest if err := json.Unmarshal(layerBytes, &zarfImageManifest); err != nil { return nil, int64(0), err } @@ -281,8 +282,10 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o if err != nil { return err } - remote.Transport.ProgressBar = message.NewProgressBar(estimatedBytes, fmt.Sprintf("Publishing %s:%s", remote.Repo().Reference.Repository, remote.Repo().Reference.Reference)) - defer remote.Transport.ProgressBar.Stop() + progressBar := message.NewProgressBar(estimatedBytes, fmt.Sprintf("Publishing %s:%s", remote.Repo().Reference.Repository, remote.Repo().Reference.Reference)) + defer progressBar.Stop() + remote.SetProgressWriter(progressBar) + defer remote.ClearProgressWriter() ref := bundle.Metadata.Version @@ -303,7 +306,7 @@ func (tp *tarballBundleProvider) PublishBundle(bundle types.UDSBundle, remote *o return err } - remote.Transport.ProgressBar.Successf("Published %s", remote.Repo().Reference) + progressBar.Successf("Published %s", remote.Repo().Reference) return nil } @@ -316,7 +319,7 @@ func (tp *tarballBundleProvider) ZarfPackageNameMap() (map[string]string, error) nameMap := make(map[string]string) for _, layer := range bundleRootManifest.Layers { - if layer.MediaType == oci.ZarfLayerMediaTypeBlob { + if layer.MediaType == zoci.ZarfLayerMediaTypeBlob { // only the uds bundle layer will have AnnotationTitle set if layer.Annotations[ocispec.AnnotationTitle] != config.BundleYAML { nameMap[layer.Annotations[config.UDSPackageNameAnnotation]] = layer.Annotations[config.ZarfPackageNameAnnotation] diff --git a/src/pkg/bundler/common.go b/src/pkg/bundler/common.go index 73611936..10a77a6f 100644 --- a/src/pkg/bundler/common.go +++ b/src/pkg/bundler/common.go @@ -5,6 +5,7 @@ package bundler import ( + "context" "errors" "fmt" "strings" @@ -14,6 +15,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/utils/helpers" + "github.com/defenseunicorns/zarf/src/pkg/zoci" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2/registry" ) @@ -44,7 +46,7 @@ func manifestAnnotationsFromMetadata(metadata *types.UDSMetadata) map[string]str } // copied from: https://github.com/defenseunicorns/zarf/blob/main/src/pkg/oci/push.go -func pushManifestConfigFromMetadata(r *oci.OrasRemote, metadata *types.UDSMetadata, build *types.UDSBuildData) (ocispec.Descriptor, error) { +func pushManifestConfigFromMetadata(ctx context.Context, r *oci.OrasRemote, metadata *types.UDSMetadata, build *types.UDSBuildData) (ocispec.Descriptor, error) { annotations := map[string]string{ ocispec.AnnotationTitle: metadata.Name, ocispec.AnnotationDescription: metadata.Description, @@ -54,11 +56,11 @@ func pushManifestConfigFromMetadata(r *oci.OrasRemote, metadata *types.UDSMetada OCIVersion: "1.0.1", Annotations: annotations, } - manifestConfigDesc, err := utils.ToOCIRemote(manifestConfig, oci.ZarfLayerMediaTypeBlob, r) + manifestConfigDesc, err := utils.ToOCIRemote(ctx, manifestConfig, zoci.ZarfLayerMediaTypeBlob, r) if err != nil { return ocispec.Descriptor{}, err } - return manifestConfigDesc, nil + return *manifestConfigDesc, nil } // copied from: https://github.com/defenseunicorns/zarf/blob/main/src/pkg/oci/utils.go diff --git a/src/pkg/bundler/fetcher/fetcher.go b/src/pkg/bundler/fetcher/fetcher.go index 2d68e2cb..bc447e0e 100644 --- a/src/pkg/bundler/fetcher/fetcher.go +++ b/src/pkg/bundler/fetcher/fetcher.go @@ -12,6 +12,7 @@ import ( "github.com/defenseunicorns/uds-cli/src/pkg/utils" "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/zoci" zarfTypes "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ocistore "oras.land/oras-go/v2/content/oci" @@ -42,16 +43,17 @@ func NewPkgFetcher(pkg types.Package, fetcherConfig Config) (Fetcher, error) { OS: oci.MultiOS, } url := fmt.Sprintf("%s:%s", pkg.Repository, pkg.Ref) - remote, err := oci.NewOrasRemote(url, platform) + remote, err := zoci.NewRemote(url, platform) if err != nil { return nil, err } - pkgRootManifest, err := remote.FetchRoot() + ctx := context.TODO() + pkgRootManifest, err := remote.FetchRoot(ctx) if err != nil { return nil, err } fetcher = &remoteFetcher{ - ctx: context.TODO(), + ctx: ctx, pkg: pkg, cfg: fetcherConfig, pkgRootManifest: pkgRootManifest, diff --git a/src/pkg/bundler/fetcher/local.go b/src/pkg/bundler/fetcher/local.go index 580cb9b2..fac4ae41 100644 --- a/src/pkg/bundler/fetcher/local.go +++ b/src/pkg/bundler/fetcher/local.go @@ -17,6 +17,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/zoci" zarfTypes "github.com/defenseunicorns/zarf/src/types" goyaml "github.com/goccy/go-yaml" av3 "github.com/mholt/archiver/v3" @@ -165,7 +166,7 @@ func (f *localFetcher) toBundle(pkg zarfTypes.ZarfPackage, pkgTmp string) ([]oci return nil, err } - mediaType := oci.ZarfLayerMediaTypeBlob + mediaType := zoci.ZarfLayerMediaTypeBlob // todo: try finding the desc with media type of image manifest, and rewrite it here! // just iterate through it's layers and add the annotations to each layer, then push to the store and add to descs @@ -247,11 +248,11 @@ func generatePkgManifest(store *ocistore.Store, descs []ocispec.Descriptor, conf SchemaVersion: 2, // historical value. does not pertain to OCI or docker version }, Config: configDesc, - MediaType: oci.ZarfLayerMediaTypeBlob, + MediaType: zoci.ZarfLayerMediaTypeBlob, Layers: descs, } - manifestDesc, err := utils.ToOCIStore(manifest, oci.ZarfLayerMediaTypeBlob, store) + manifestDesc, err := utils.ToOCIStore(manifest, zoci.ZarfLayerMediaTypeBlob, store) if err != nil { return ocispec.Descriptor{}, err } diff --git a/src/pkg/bundler/fetcher/remote.go b/src/pkg/bundler/fetcher/remote.go index ab11ec10..f7a92972 100644 --- a/src/pkg/bundler/fetcher/remote.go +++ b/src/pkg/bundler/fetcher/remote.go @@ -11,7 +11,6 @@ import ( "os" "path/filepath" "strings" - "sync" "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/uds-cli/src/pkg/cache" @@ -20,6 +19,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/zoci" zarfTypes "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" @@ -29,8 +29,8 @@ type remoteFetcher struct { ctx context.Context pkg types.Package cfg Config - pkgRootManifest *oci.ZarfOCIManifest - remote *oci.OrasRemote + pkgRootManifest *oci.Manifest + remote *zoci.Remote } // Fetch fetches a Zarf pkg and puts it into a local bundle @@ -53,12 +53,12 @@ func (f *remoteFetcher) Fetch() ([]ocispec.Descriptor, error) { if err != nil { return nil, err } - err = utils.FetchLayerAndStore(layerDesc, f.remote, f.cfg.Store) + err = utils.FetchLayerAndStore(f.ctx, layerDesc, f.remote.OrasRemote, f.cfg.Store) if err != nil { return nil, err } // ensure media type is Zarf blob for layers in the bundle's root manifest - layerDesc.MediaType = oci.ZarfLayerMediaTypeBlob + layerDesc.MediaType = zoci.ZarfLayerMediaTypeBlob // add package name annotations annotations := make(map[string]string) @@ -73,7 +73,7 @@ func (f *remoteFetcher) Fetch() ([]ocispec.Descriptor, error) { f.cfg.BundleRootManifest.Layers = append(f.cfg.BundleRootManifest.Layers, layerDesc) zarfRootLayerAdded = true - } else if layerDesc.MediaType == oci.ZarfConfigMediaType { + } else if layerDesc.MediaType == zoci.ZarfConfigMediaType { // read in and unmarshal zarf config jsonData, err := os.ReadFile(filepath.Join(f.cfg.TmpDstDir, config.BlobsDir, layerDesc.Digest.Encoded())) if err != nil { @@ -100,7 +100,7 @@ func (f *remoteFetcher) Fetch() ([]ocispec.Descriptor, error) { func (f *remoteFetcher) layersToLocalBundle(spinner *message.Spinner, currentPackageIter int, totalPackages int) ([]ocispec.Descriptor, error) { spinner.Updatef("Fetching %s package layer metadata (package %d of %d)", f.pkg.Name, currentPackageIter, totalPackages) // get only the layers that are required by the components - layersToCopy, err := utils.GetZarfLayers(f.remote, f.pkg, f.pkgRootManifest) + layersToCopy, err := utils.GetZarfLayers(f.ctx, *f.remote, f.pkg, f.pkgRootManifest) if err != nil { return nil, err } @@ -145,10 +145,7 @@ func (f *remoteFetcher) remoteToLocal(layersToCopy []ocispec.Descriptor) ([]ocis // copy Zarf pkg copyOpts := utils.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency) // Create a thread to update a progress bar as we save the package to disk - doneSaving := make(chan int) - errChan := make(chan int) - var wg sync.WaitGroup - wg.Add(1) + doneSaving := make(chan error) // Grab tmpDirSize and add it to the estimatedBytes, otherwise the progress bar will be off // because as multiple packages are pulled into the tmpDir, RenderProgressBarForLocalDirWrite continues to @@ -158,14 +155,13 @@ func (f *remoteFetcher) remoteToLocal(layersToCopy []ocispec.Descriptor) ([]ocis return nil, err } - go zarfUtils.RenderProgressBarForLocalDirWrite(f.cfg.TmpDstDir, estimatedBytes+tmpDirSize, &wg, doneSaving, errChan, fmt.Sprintf("Pulling bundle: %s", f.pkg.Name), fmt.Sprintf("Successfully pulled package: %s", f.pkg.Name)) + go zarfUtils.RenderProgressBarForLocalDirWrite(f.cfg.TmpDstDir, estimatedBytes+tmpDirSize, doneSaving, fmt.Sprintf("Pulling bundle: %s", f.pkg.Name), fmt.Sprintf("Successfully pulled package: %s", f.pkg.Name)) rootPkgDesc, err := oras.Copy(context.TODO(), f.remote.Repo(), f.remote.Repo().Reference.String(), f.cfg.Store, "", copyOpts) + doneSaving <- err + <-doneSaving if err != nil { - errChan <- 1 return nil, err } - doneSaving <- 1 - wg.Wait() // grab pkg root manifest for archiving descsToBundle = append(descsToBundle, rootPkgDesc) @@ -196,7 +192,7 @@ func (f *remoteFetcher) GetPkgMetadata() (zarfTypes.ZarfPackage, error) { OS: oci.MultiOS, } url := fmt.Sprintf("%s:%s", f.pkg.Repository, f.pkg.Ref) - remote, err := oci.NewOrasRemote(url, platform) + remote, err := zoci.NewRemote(url, platform) if err != nil { return zarfTypes.ZarfPackage{}, err } @@ -204,7 +200,7 @@ func (f *remoteFetcher) GetPkgMetadata() (zarfTypes.ZarfPackage, error) { if err != nil { return zarfTypes.ZarfPackage{}, fmt.Errorf("bundler unable to create temp directory: %w", err) } - if _, err := remote.PullPackageMetadata(tmpDir); err != nil { + if _, err := remote.PullPackageMetadata(f.ctx, tmpDir); err != nil { return zarfTypes.ZarfPackage{}, err } zarfYAML := zarfTypes.ZarfPackage{} diff --git a/src/pkg/bundler/localbundle.go b/src/pkg/bundler/localbundle.go index e02abe63..84734e29 100644 --- a/src/pkg/bundler/localbundle.go +++ b/src/pkg/bundler/localbundle.go @@ -18,6 +18,7 @@ import ( "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/zoci" goyaml "github.com/goccy/go-yaml" "github.com/mholt/archiver/v4" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -173,7 +174,7 @@ func pushBundleYAMLToStore(ctx context.Context, store *ocistore.Store, bundle *t if err != nil { return ocispec.Descriptor{}, err } - bundleYamlDesc := content.NewDescriptorFromBytes(oci.ZarfLayerMediaTypeBlob, bundleYAMLBytes) + bundleYamlDesc := content.NewDescriptorFromBytes(zoci.ZarfLayerMediaTypeBlob, bundleYAMLBytes) bundleYamlDesc.Annotations = map[string]string{ ocispec.AnnotationTitle: config.BundleYAML, } @@ -196,7 +197,7 @@ func pushManifestConfig(store *ocistore.Store, metadata types.UDSMetadata, build OCIVersion: "1.0.1", Annotations: annotations, } - manifestConfigDesc, err := utils.ToOCIStore(manifestConfig, oci.ZarfLayerMediaTypeBlob, store) + manifestConfigDesc, err := utils.ToOCIStore(manifestConfig, zoci.ZarfLayerMediaTypeBlob, store) if err != nil { return ocispec.Descriptor{}, err } @@ -272,7 +273,7 @@ jobLoop: } func pushBundleSignature(ctx context.Context, store *ocistore.Store, signature []byte) (ocispec.Descriptor, error) { - signatureDesc := content.NewDescriptorFromBytes(oci.ZarfLayerMediaTypeBlob, signature) + signatureDesc := content.NewDescriptorFromBytes(zoci.ZarfLayerMediaTypeBlob, signature) err := store.Push(ctx, signatureDesc, bytes.NewReader(signature)) if err != nil { return ocispec.Descriptor{}, err diff --git a/src/pkg/bundler/pusher/remote.go b/src/pkg/bundler/pusher/remote.go index 9abca0df..d2b39204 100644 --- a/src/pkg/bundler/pusher/remote.go +++ b/src/pkg/bundler/pusher/remote.go @@ -14,6 +14,7 @@ import ( "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/zoci" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -26,9 +27,9 @@ type RemotePusher struct { // Config contains the configuration for the remote pusher type Config struct { - PkgRootManifest *oci.ZarfOCIManifest - RemoteSrc *oci.OrasRemote - RemoteDst *oci.OrasRemote + PkgRootManifest *oci.Manifest + RemoteSrc zoci.Remote + RemoteDst zoci.Remote PkgIter int NumPkgs int Bundle *types.UDSBundle @@ -47,12 +48,12 @@ func (p *RemotePusher) Push() (ocispec.Descriptor, error) { } // ensure media type is a Zarf blob and append to bundle root manifest - zarfManifestDesc.MediaType = oci.ZarfLayerMediaTypeBlob + zarfManifestDesc.MediaType = zoci.ZarfLayerMediaTypeBlob url := fmt.Sprintf("%s:%s", p.pkg.Repository, p.pkg.Ref) message.Debugf("Pushed %s sub-manifest into %s: %s", url, p.cfg.RemoteDst.Repo().Reference, message.JSONValue(zarfManifestDesc)) // add package name annotations to zarf manifest - zarfYamlFile, err := p.cfg.RemoteSrc.FetchZarfYAML() + zarfYamlFile, err := p.cfg.RemoteSrc.FetchZarfYAML(p.ctx) if err != nil { return ocispec.Descriptor{}, err } @@ -75,11 +76,11 @@ func (p *RemotePusher) Push() (ocispec.Descriptor, error) { // PushManifest pushes the Zarf pkg's manifest to either a local or remote bundle func (p *RemotePusher) PushManifest() (ocispec.Descriptor, error) { var zarfManifestDesc ocispec.Descriptor - desc, err := utils.ToOCIRemote(p.cfg.PkgRootManifest, oci.ZarfLayerMediaTypeBlob, p.cfg.RemoteDst) + desc, err := utils.ToOCIRemote(p.ctx, p.cfg.PkgRootManifest, zoci.ZarfLayerMediaTypeBlob, p.cfg.RemoteDst.OrasRemote) if err != nil { return ocispec.Descriptor{}, err } - zarfManifestDesc = desc + zarfManifestDesc = *desc return zarfManifestDesc, nil } @@ -87,7 +88,7 @@ func (p *RemotePusher) PushManifest() (ocispec.Descriptor, error) { func (p *RemotePusher) LayersToRemoteBundle(spinner *message.Spinner, currentPackageIter int, totalPackages int) ([]ocispec.Descriptor, error) { spinner.Updatef("Fetching %s package layer metadata (package %d of %d)", p.pkg.Name, currentPackageIter, totalPackages) // get only the layers that are required by the components - layersToCopy, err := utils.GetZarfLayers(p.cfg.RemoteSrc, p.pkg, p.cfg.PkgRootManifest) + layersToCopy, err := utils.GetZarfLayers(p.ctx, p.cfg.RemoteSrc, p.pkg, p.cfg.PkgRootManifest) if err != nil { return nil, err } @@ -108,17 +109,7 @@ func (p *RemotePusher) remoteToRemote(layersToCopy []ocispec.Descriptor) error { if srcRef.Registry != dstRef.Registry { message.Debugf("Streaming layers from %s --> %s", srcRef, dstRef) - // filterLayers returns true if the layer is in the list of layers to copy, this allows for - // copying only the layers that are required by the required + specified optional components - filterLayers := func(d ocispec.Descriptor) bool { - for _, layer := range layersToCopy { - if layer.Digest == d.Digest { - return true - } - } - return false - } - if err := oci.CopyPackage(p.ctx, p.cfg.RemoteSrc, p.cfg.RemoteDst, filterLayers, config.CommonOptions.OCIConcurrency); err != nil { + if err := zoci.CopyPackage(p.ctx, &p.cfg.RemoteSrc, &p.cfg.RemoteDst, config.CommonOptions.OCIConcurrency); err != nil { return err } } else { diff --git a/src/pkg/bundler/remotebundle.go b/src/pkg/bundler/remotebundle.go index 4df60590..f3955f45 100644 --- a/src/pkg/bundler/remotebundle.go +++ b/src/pkg/bundler/remotebundle.go @@ -5,6 +5,7 @@ package bundler import ( + "context" "fmt" "github.com/defenseunicorns/uds-cli/src/config" @@ -13,6 +14,7 @@ import ( "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/zoci" goyaml "github.com/goccy/go-yaml" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -29,6 +31,7 @@ type RemoteBundle struct { bundle *types.UDSBundle tmpDstDir string output string + ctx context.Context } // NewRemoteBundle creates a new remote bundle @@ -37,6 +40,7 @@ func NewRemoteBundle(opts *RemoteBundleOpts) *RemoteBundle { bundle: opts.Bundle, tmpDstDir: opts.TmpDstDir, output: opts.Output, + ctx: context.TODO(), } } @@ -54,7 +58,7 @@ func (r *RemoteBundle) create(signature []byte) error { } // create the bundle remote - bundleRemote, err := oci.NewOrasRemote(ref, platform) + bundleRemote, err := zoci.NewRemote(ref, platform) if err != nil { return err } @@ -68,19 +72,19 @@ func (r *RemoteBundle) create(signature []byte) error { rootManifest := ocispec.Manifest{} pusherConfig := pusher.Config{ Bundle: bundle, - RemoteDst: bundleRemote, + RemoteDst: *bundleRemote, NumPkgs: len(bundle.Packages), } for i, pkg := range bundle.Packages { // todo: can leave this block here or move to pusher.NewPkgPusher (would be closer to NewPkgFetcher pattern) pkgURL := fmt.Sprintf("%s:%s", pkg.Repository, pkg.Ref) - src, err := oci.NewOrasRemote(pkgURL, platform) + src, err := zoci.NewRemote(pkgURL, platform) if err != nil { return err } - pusherConfig.RemoteSrc = src - pkgRootManifest, err := src.FetchRoot() + pusherConfig.RemoteSrc = *src + pkgRootManifest, err := src.FetchRoot(r.ctx) if err != nil { return err } @@ -100,7 +104,7 @@ func (r *RemoteBundle) create(signature []byte) error { if err != nil { return err } - bundleYamlDesc, err := bundleRemote.PushLayer(bundleYamlBytes, oci.ZarfLayerMediaTypeBlob) + bundleYamlDesc, err := bundleRemote.PushLayer(r.ctx, bundleYamlBytes, zoci.ZarfLayerMediaTypeBlob) if err != nil { return err } @@ -109,23 +113,23 @@ func (r *RemoteBundle) create(signature []byte) error { } message.Debug("Pushed", config.BundleYAML+":", message.JSONValue(bundleYamlDesc)) - rootManifest.Layers = append(rootManifest.Layers, bundleYamlDesc) + rootManifest.Layers = append(rootManifest.Layers, *bundleYamlDesc) // push the bundle's signature if len(signature) > 0 { - bundleYamlSigDesc, err := bundleRemote.PushLayer(signature, oci.ZarfLayerMediaTypeBlob) + bundleYamlSigDesc, err := bundleRemote.PushLayer(r.ctx, signature, zoci.ZarfLayerMediaTypeBlob) if err != nil { return err } bundleYamlSigDesc.Annotations = map[string]string{ ocispec.AnnotationTitle: config.BundleYAMLSignature, } - rootManifest.Layers = append(rootManifest.Layers, bundleYamlSigDesc) + rootManifest.Layers = append(rootManifest.Layers, *bundleYamlSigDesc) message.Debug("Pushed", config.BundleYAMLSignature+":", message.JSONValue(bundleYamlSigDesc)) } // push the bundle manifest config - configDesc, err := pushManifestConfigFromMetadata(bundleRemote, &bundle.Metadata, &bundle.Build) + configDesc, err := pushManifestConfigFromMetadata(r.ctx, bundleRemote.OrasRemote, &bundle.Metadata, &bundle.Build) if err != nil { return err } @@ -133,7 +137,7 @@ func (r *RemoteBundle) create(signature []byte) error { message.Debug("Pushed config:", message.JSONValue(configDesc)) // check for existing index - index, err := utils.GetIndex(bundleRemote, dstRef.String()) + index, err := utils.GetIndex(bundleRemote.OrasRemote, dstRef.String()) if err != nil { return err } @@ -142,13 +146,13 @@ func (r *RemoteBundle) create(signature []byte) error { rootManifest.Config = configDesc rootManifest.SchemaVersion = 2 rootManifest.Annotations = manifestAnnotationsFromMetadata(&bundle.Metadata) // maps to registry UI - rootManifestDesc, err := utils.ToOCIRemote(rootManifest, ocispec.MediaTypeImageManifest, bundleRemote) + rootManifestDesc, err := utils.ToOCIRemote(r.ctx, rootManifest, ocispec.MediaTypeImageManifest, bundleRemote.OrasRemote) if err != nil { return err } // create or update, then push index.json - err = utils.UpdateIndex(index, bundleRemote, bundle, rootManifestDesc) + err = utils.UpdateIndex(index, bundleRemote.OrasRemote, bundle, *rootManifestDesc) if err != nil { return err } diff --git a/src/pkg/sources/new.go b/src/pkg/sources/new.go index ac55ce3b..1a3cd677 100644 --- a/src/pkg/sources/new.go +++ b/src/pkg/sources/new.go @@ -5,11 +5,13 @@ package sources import ( + "context" "strings" "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/zarf/src/pkg/oci" zarfSources "github.com/defenseunicorns/zarf/src/pkg/packager/sources" + "github.com/defenseunicorns/zarf/src/pkg/zoci" zarfTypes "github.com/defenseunicorns/zarf/src/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -30,7 +32,7 @@ func New(pkgLocation string, pkgName string, opts zarfTypes.ZarfPackageOptions, Architecture: config.GetArch(), OS: oci.MultiOS, } - remote, err := oci.NewOrasRemote(pkgLocation, platform) + remote, err := zoci.NewRemote(pkgLocation, platform) if err != nil { return nil, err } @@ -39,7 +41,8 @@ func New(pkgLocation string, pkgName string, opts zarfTypes.ZarfPackageOptions, PkgOpts: &opts, PkgManifestSHA: sha, TmpDir: opts.PackageSource, - Remote: remote, + Remote: remote.OrasRemote, + ctx: context.TODO(), } } return source, nil diff --git a/src/pkg/sources/remote.go b/src/pkg/sources/remote.go index 08d3c2c1..f45dc0ce 100644 --- a/src/pkg/sources/remote.go +++ b/src/pkg/sources/remote.go @@ -10,7 +10,6 @@ import ( "os" "path/filepath" "strings" - "sync" "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/uds-cli/src/pkg/cache" @@ -20,6 +19,7 @@ import ( "github.com/defenseunicorns/zarf/src/pkg/oci" "github.com/defenseunicorns/zarf/src/pkg/packager/sources" zarfUtils "github.com/defenseunicorns/zarf/src/pkg/utils" + "github.com/defenseunicorns/zarf/src/pkg/zoci" zarfTypes "github.com/defenseunicorns/zarf/src/types" goyaml "github.com/goccy/go-yaml" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -35,6 +35,7 @@ type RemoteBundle struct { TmpDir string Remote *oci.OrasRemote isPartial bool + ctx context.Context } // LoadPackage loads a Zarf package from a remote bundle @@ -81,7 +82,7 @@ func (r *RemoteBundle) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) // LoadPackageMetadata loads a Zarf package's metadata from a remote bundle func (r *RemoteBundle) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ bool) (err error) { - root, err := r.Remote.FetchRoot() + root, err := r.Remote.FetchRoot(r.ctx) if err != nil { return err } @@ -91,7 +92,7 @@ func (r *RemoteBundle) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ b } // look at Zarf pkg manifest, grab zarf.yaml desc and download it - pkgManifest, err := r.Remote.FetchManifest(pkgManifestDesc) + pkgManifest, err := r.Remote.FetchManifest(r.ctx, pkgManifestDesc) if err != nil { return err } @@ -103,7 +104,7 @@ func (r *RemoteBundle) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ b break } } - zarfYAMLBytes, err := r.Remote.FetchLayer(zarfYAMLDesc) + zarfYAMLBytes, err := r.Remote.FetchLayer(r.ctx, zarfYAMLDesc) if err != nil { return err } @@ -120,7 +121,7 @@ func (r *RemoteBundle) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ b var checksumLayer ocispec.Descriptor for _, layer := range pkgManifest.Layers { if layer.Annotations[ocispec.AnnotationTitle] == config.ChecksumsTxt { - checksumBytes, err := r.Remote.FetchLayer(layer) + checksumBytes, err := r.Remote.FetchLayer(r.ctx, layer) if err != nil { return err } @@ -146,7 +147,7 @@ func (r *RemoteBundle) Collect(_ string) (string, error) { // downloadPkgFromRemoteBundle downloads a Zarf package from a remote bundle func (r *RemoteBundle) downloadPkgFromRemoteBundle() ([]ocispec.Descriptor, error) { - rootManifest, err := r.Remote.FetchRoot() + rootManifest, err := r.Remote.FetchRoot(r.ctx) if err != nil { return nil, err } @@ -156,8 +157,8 @@ func (r *RemoteBundle) downloadPkgFromRemoteBundle() ([]ocispec.Descriptor, erro return nil, fmt.Errorf("package %s does not exist in this bundle", r.PkgManifestSHA) } // hack Zarf media type so that FetchManifest works - pkgManifestDesc.MediaType = oci.ZarfLayerMediaTypeBlob - pkgManifest, err := r.Remote.FetchManifest(pkgManifestDesc) + pkgManifestDesc.MediaType = zoci.ZarfLayerMediaTypeBlob + pkgManifest, err := r.Remote.FetchManifest(r.ctx, pkgManifestDesc) if err != nil || pkgManifest == nil { return nil, err } @@ -201,18 +202,14 @@ func (r *RemoteBundle) downloadPkgFromRemoteBundle() ([]ocispec.Descriptor, erro // copy zarf pkg to local store copyOpts := utils.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency) - doneSaving := make(chan int) - errChan := make(chan int) - var wg sync.WaitGroup - wg.Add(1) - go zarfUtils.RenderProgressBarForLocalDirWrite(r.TmpDir, estimatedBytes, &wg, doneSaving, errChan, fmt.Sprintf("Pulling bundled Zarf pkg: %s", r.PkgName), fmt.Sprintf("Successfully pulled package: %s", r.PkgName)) + doneSaving := make(chan error) + go zarfUtils.RenderProgressBarForLocalDirWrite(r.TmpDir, estimatedBytes, doneSaving, fmt.Sprintf("Pulling bundled Zarf pkg: %s", r.PkgName), fmt.Sprintf("Successfully pulled package: %s", r.PkgName)) _, err = oras.Copy(context.TODO(), r.Remote.Repo(), r.Remote.Repo().Reference.String(), store, "", copyOpts) + doneSaving <- err + <-doneSaving if err != nil { - errChan <- 1 return nil, err } - doneSaving <- 1 - wg.Wait() // need to substract 1 from layersInBundle because it includes the pkgManifestDesc and pkgManifest.Layers does not if len(pkgManifest.Layers) != len(layersInBundle)-1 { diff --git a/src/pkg/sources/tarball.go b/src/pkg/sources/tarball.go index eabeaaa9..93afb573 100644 --- a/src/pkg/sources/tarball.go +++ b/src/pkg/sources/tarball.go @@ -92,7 +92,7 @@ func (t *TarballBundle) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ return err } - var imageManifest oci.ZarfOCIManifest + var imageManifest oci.Manifest if err := format.Extract(ctx, sourceArchive, []string{filepath.Join(config.BlobsDir, t.PkgManifestSHA)}, utils.ExtractJSON(&imageManifest)); err != nil { return err } @@ -189,7 +189,7 @@ func (t *TarballBundle) extractPkgFromBundle() ([]string, error) { return nil, err } - var manifest oci.ZarfOCIManifest + var manifest oci.Manifest if err := format.Extract(context.TODO(), sourceArchive, []string{filepath.Join(config.BlobsDir, t.PkgManifestSHA)}, utils.ExtractJSON(&manifest)); err != nil { if err := sourceArchive.Close(); err != nil { return nil, err diff --git a/src/pkg/utils/oci.go b/src/pkg/utils/oci.go index e0632d0e..a0cae61d 100644 --- a/src/pkg/utils/oci.go +++ b/src/pkg/utils/oci.go @@ -17,6 +17,7 @@ import ( "github.com/defenseunicorns/uds-cli/src/types" "github.com/defenseunicorns/zarf/src/pkg/message" "github.com/defenseunicorns/zarf/src/pkg/oci" + "github.com/defenseunicorns/zarf/src/pkg/zoci" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "oras.land/oras-go/v2" "oras.land/oras-go/v2/content" @@ -25,12 +26,12 @@ import ( ) // FetchLayerAndStore fetches a remote layer and copies it to a local store -func FetchLayerAndStore(layerDesc ocispec.Descriptor, remoteRepo *oci.OrasRemote, localStore *ocistore.Store) error { - layerBytes, err := remoteRepo.FetchLayer(layerDesc) +func FetchLayerAndStore(ctx context.Context, layerDesc ocispec.Descriptor, remoteRepo *oci.OrasRemote, localStore *ocistore.Store) error { + layerBytes, err := remoteRepo.FetchLayer(ctx, layerDesc) if err != nil { return err } - rootPkgDescBytes := content.NewDescriptorFromBytes(oci.ZarfLayerMediaTypeBlob, layerBytes) + rootPkgDescBytes := content.NewDescriptorFromBytes(zoci.ZarfLayerMediaTypeBlob, layerBytes) err = localStore.Push(context.TODO(), rootPkgDescBytes, bytes.NewReader(layerBytes)) return err } @@ -52,23 +53,24 @@ func ToOCIStore(t any, mediaType string, store *ocistore.Store) (ocispec.Descrip } // ToOCIRemote takes an arbitrary type, typically a struct, marshals it into JSON and store it in a remote OCI store -func ToOCIRemote(t any, mediaType string, remote *oci.OrasRemote) (ocispec.Descriptor, error) { +func ToOCIRemote(ctx context.Context, t any, mediaType string, remote *oci.OrasRemote) (*ocispec.Descriptor, error) { b, err := json.Marshal(t) if err != nil { - return ocispec.Descriptor{}, err + return &ocispec.Descriptor{}, err } - var layerDesc ocispec.Descriptor + var layerDesc *ocispec.Descriptor // if image manifest media type, push to Manifests(), otherwise normal pushLayer() if mediaType == ocispec.MediaTypeImageManifest { - layerDesc = content.NewDescriptorFromBytes(ocispec.MediaTypeImageManifest, b) - if err := remote.Repo().Manifests().PushReference(context.TODO(), layerDesc, bytes.NewReader(b), remote.Repo().Reference.String()); err != nil { - return ocispec.Descriptor{}, fmt.Errorf("failed to push manifest: %w", err) + descriptorFromBytes := content.NewDescriptorFromBytes(ocispec.MediaTypeImageManifest, b) + layerDesc = &descriptorFromBytes + if err := remote.Repo().Manifests().PushReference(context.TODO(), *layerDesc, bytes.NewReader(b), remote.Repo().Reference.String()); err != nil { + return &ocispec.Descriptor{}, fmt.Errorf("failed to push manifest: %w", err) } } else { - layerDesc, err = remote.PushLayer(b, mediaType) + layerDesc, err = remote.PushLayer(ctx, b, mediaType) if err != nil { - return ocispec.Descriptor{}, err + return &ocispec.Descriptor{}, err } } @@ -110,7 +112,7 @@ func CreateCopyOpts(layersToPull []ocispec.Descriptor, concurrency int) oras.Cop return []ocispec.Descriptor{node}, nil } } - } else if desc.MediaType == oci.ZarfLayerMediaTypeBlob && !hasTitleAnnotation { + } else if desc.MediaType == zoci.ZarfLayerMediaTypeBlob && !hasTitleAnnotation { // This if block is for used for finding successors from bundle root manifests during bundle pull/publish ops; // note that ptrs to the Zarf pkg image manifests won't have title annotations, and will follow this code path // adopted from the content.Successors() fn in oras @@ -118,7 +120,7 @@ func CreateCopyOpts(layersToPull []ocispec.Descriptor, concurrency int) oras.Cop if err != nil { return nil, err } - var manifest oci.ZarfOCIManifest + var manifest oci.Manifest if err := json.Unmarshal(layerBytes, &manifest); err != nil { return nil, err } @@ -255,14 +257,14 @@ func EnsureOCIPrefix(source string) string { } // GetZarfLayers grabs the necessary Zarf pkg layers from a remote OCI registry -func GetZarfLayers(remote *oci.OrasRemote, pkg types.Package, pkgRootManifest *oci.ZarfOCIManifest) ([]ocispec.Descriptor, error) { - layersFromComponents, err := remote.LayersFromRequestedComponents(pkg.OptionalComponents) +func GetZarfLayers(ctx context.Context, remote zoci.Remote, pkg types.Package, pkgRootManifest *oci.Manifest) ([]ocispec.Descriptor, error) { + layersFromComponents, err := remote.LayersFromRequestedComponents(ctx, pkg.OptionalComponents) if err != nil { return nil, err } // get the layers that are always pulled var metadataLayers []ocispec.Descriptor - for _, path := range oci.PackageAlwaysPull { + for _, path := range zoci.PackageAlwaysPull { layer := pkgRootManifest.Locate(path) if !oci.IsEmptyDescriptor(layer) { metadataLayers = append(metadataLayers, layer) From 74cd5361fc51c1e50c9a0220fd4cdc60d760631e Mon Sep 17 00:00:00 2001 From: Darcy Cleaver Date: Thu, 14 Mar 2024 08:53:28 -0600 Subject: [PATCH 2/7] debugging --- go.mod | 12 ++++-------- go.sum | 13 ++++++++----- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index dc575e7f..f36ff6ac 100644 --- a/go.mod +++ b/go.mod @@ -2,15 +2,11 @@ module github.com/defenseunicorns/uds-cli go 1.21.6 -replace github.com/defenseunicorns/zarf => /Users/darcy/leapfrog/gh-repos/defenseunicorns/zarf - -replace github.com/defenseunicorns/maru-runner => /Users/darcy/leapfrog/gh-repos/defenseunicorns/maru-runner - require ( github.com/AlecAivazis/survey/v2 v2.3.7 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b - github.com/defenseunicorns/maru-runner v0.0.1 - github.com/defenseunicorns/zarf v0.32.4 + github.com/defenseunicorns/maru-runner v0.0.2 + github.com/defenseunicorns/zarf v0.32.5 github.com/fsnotify/fsnotify v1.7.0 github.com/goccy/go-yaml v1.11.3 github.com/mholt/archiver/v3 v3.5.1 @@ -216,7 +212,7 @@ require ( github.com/go-git/go-git/v5 v5.11.0 // indirect github.com/go-gorp/gorp/v3 v3.1.0 // indirect github.com/go-ini/ini v1.67.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.1 // indirect + github.com/go-jose/go-jose/v3 v3.0.3 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/analysis v0.22.0 // indirect @@ -477,7 +473,7 @@ require ( google.golang.org/grpc v1.61.0 // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/evanphx/json-patch.v5 v5.6.0 // indirect - gopkg.in/go-jose/go-jose.v2 v2.6.1 // indirect + gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 60ddddad..b5748dc9 100644 --- a/go.sum +++ b/go.sum @@ -598,6 +598,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX8ATG8oKsE= github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c= +github.com/defenseunicorns/maru-runner v0.0.2 h1:aZ/MCryY3Dp+u14nhGrvOXVnZI3DYY3ytjFdfHnyqY4= +github.com/defenseunicorns/maru-runner v0.0.2/go.mod h1:3K+JeLpud+rb8vC+nPFaTNjhqW40++6qFKKVTBEEzQM= +github.com/defenseunicorns/zarf v0.32.5 h1:g7hQPCUEd8PlK4s6f+e7naCS9q85ghWLjegEbcgUvPs= +github.com/defenseunicorns/zarf v0.32.5/go.mod h1:dDIYZdh4XGkI/A6EpZHEBzQaQjw0hSbxJs4MQFOSqLc= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da h1:ZOjWpVsFZ06eIhnh4mkaceTiVoktdU67+M7KDHJ268M= github.com/deitch/magic v0.0.0-20230404182410-1ff89d7342da/go.mod h1:B3tI9iGHi4imdLi4Asdha1Sc6feLMTfPLXh9IUYmysk= github.com/depcheck-test/depcheck-test v0.0.0-20220607135614-199033aaa936 h1:foGzavPWwtoyBvjWyKJYDYsyzy+23iBV7NKTwdk+LRY= @@ -761,8 +765,8 @@ github.com/go-gorp/gorp/v3 v3.1.0 h1:ItKF/Vbuj31dmV4jxA1qblpSwkl9g1typ24xoe70IGs github.com/go-gorp/gorp/v3 v3.1.0/go.mod h1:dLEjIyyRNiXvNZ8PSmzpt1GsWAUK8kjVhEpjH8TixEw= github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA= -github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= +github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= +github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -1768,7 +1772,6 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191219195013-becbf705a915/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -2394,8 +2397,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/evanphx/json-patch.v5 v5.6.0 h1:BMT6KIwBD9CaU91PJCZIe46bDmBWa9ynTQgJIOpfQBk= gopkg.in/evanphx/json-patch.v5 v5.6.0/go.mod h1:/kvTRh1TVm5wuM6OkHxqXtE/1nUZZpihg29RtuIyfvk= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/go-jose/go-jose.v2 v2.6.1 h1:qEzJlIDmG9q5VO0M/o8tGS65QMHMS1w01TQJB1VPJ4U= -gopkg.in/go-jose/go-jose.v2 v2.6.1/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= +gopkg.in/go-jose/go-jose.v2 v2.6.3 h1:nt80fvSDlhKWQgSWyHyy5CfmlQr+asih51R8PTWNKKs= +gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.56.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= From cacea5d5d6639896a8d0b7da28bdde969281c4b5 Mon Sep 17 00:00:00 2001 From: Darcy Cleaver Date: Thu, 14 Mar 2024 12:47:28 -0600 Subject: [PATCH 3/7] update init bundle zarf version --- src/test/bundles/04-init/uds-bundle.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/bundles/04-init/uds-bundle.yaml b/src/test/bundles/04-init/uds-bundle.yaml index a416966b..7ecf2a1e 100644 --- a/src/test/bundles/04-init/uds-bundle.yaml +++ b/src/test/bundles/04-init/uds-bundle.yaml @@ -10,12 +10,12 @@ packages: - name: init path: "../../packages" # renovate: datasource=github-tags depName=defenseunicorns/zarf - ref: v0.32.4 + ref: v0.32.5 optionalComponents: - git-server - name: init repository: ghcr.io/defenseunicorns/packages/init # renovate: datasource=github-tags depName=defenseunicorns/zarf - ref: v0.32.4 + ref: v0.32.5 optionalComponents: - git-server From 7d37786549d27d663b349e370fedcaee8163e4c5 Mon Sep 17 00:00:00 2001 From: Darcy Cleaver Date: Thu, 14 Mar 2024 15:15:02 -0600 Subject: [PATCH 4/7] add retries for pkg options --- src/pkg/bundle/deploy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/bundle/deploy.go b/src/pkg/bundle/deploy.go index 9792d8eb..58eefe05 100644 --- a/src/pkg/bundle/deploy.go +++ b/src/pkg/bundle/deploy.go @@ -160,6 +160,7 @@ func deployPackages(packages []types.Package, resume bool, b *Bundle, zarfPackag OptionalComponents: strings.Join(pkg.OptionalComponents, ","), PublicKeyPath: publicKeyPath, SetVariables: pkgVars, + Retries: 1, } valuesOverrides, err := b.loadChartOverrides(pkg, pkgVars) From 60e750d3435dd0ac5df223266dc5602fedf48c0d Mon Sep 17 00:00:00 2001 From: Darcy Cleaver Date: Thu, 14 Mar 2024 20:46:34 -0600 Subject: [PATCH 5/7] fix remote to remote bug when creating to registry --- src/pkg/bundler/pusher/remote.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/pkg/bundler/pusher/remote.go b/src/pkg/bundler/pusher/remote.go index d2b39204..35bd36eb 100644 --- a/src/pkg/bundler/pusher/remote.go +++ b/src/pkg/bundler/pusher/remote.go @@ -108,8 +108,17 @@ func (p *RemotePusher) remoteToRemote(layersToCopy []ocispec.Descriptor) error { // stream copy if different registry if srcRef.Registry != dstRef.Registry { message.Debugf("Streaming layers from %s --> %s", srcRef, dstRef) - - if err := zoci.CopyPackage(p.ctx, &p.cfg.RemoteSrc, &p.cfg.RemoteDst, config.CommonOptions.OCIConcurrency); err != nil { + // filterLayers returns true if the layer is in the list of layers to copy, this allows for + // copying only the layers that are required by the required + specified optional components + filterLayers := func(d ocispec.Descriptor) bool { + for _, layer := range layersToCopy { + if layer.Digest == d.Digest { + return true + } + } + return false + } + if err := oci.Copy(p.ctx, p.cfg.RemoteSrc.OrasRemote, p.cfg.RemoteDst.OrasRemote, filterLayers, config.CommonOptions.OCIConcurrency, nil); err != nil { return err } } else { From 47cdea02c9f6551bd63fb7cea938023b32a2d44f Mon Sep 17 00:00:00 2001 From: Darcy Cleaver Date: Fri, 15 Mar 2024 13:05:57 -0600 Subject: [PATCH 6/7] addressing pr comments and refactor context usage --- src/pkg/bundle/deploy.go | 6 ++---- src/pkg/bundle/inspect.go | 7 ++----- src/pkg/bundle/provider.go | 5 +++-- src/pkg/bundle/publish.go | 3 +-- src/pkg/bundle/pull.go | 6 +++--- src/pkg/bundle/remote.go | 29 ++++++++++++++++------------- src/pkg/bundle/remove.go | 6 ++---- src/pkg/bundler/common.go | 5 ++--- src/pkg/bundler/fetcher/fetcher.go | 1 - src/pkg/bundler/fetcher/remote.go | 11 ++++++----- src/pkg/bundler/localbundle.go | 10 ++++++---- src/pkg/bundler/pusher/remote.go | 17 +++++++++-------- src/pkg/bundler/remotebundle.go | 14 +++++++------- src/pkg/sources/new.go | 2 -- src/pkg/sources/remote.go | 19 ++++++++++--------- src/pkg/utils/oci.go | 17 ++++++++++------- 16 files changed, 79 insertions(+), 79 deletions(-) diff --git a/src/pkg/bundle/deploy.go b/src/pkg/bundle/deploy.go index 58eefe05..96a8f4fe 100644 --- a/src/pkg/bundle/deploy.go +++ b/src/pkg/bundle/deploy.go @@ -5,7 +5,6 @@ package bundle import ( - "context" "encoding/json" "fmt" "os" @@ -37,7 +36,6 @@ var templatedVarRegex = regexp.MustCompile(`\${([^}]+)}`) // Deploy deploys a bundle func (b *Bundle) Deploy() error { - ctx := context.TODO() pterm.Println() metadataSpinner := message.NewProgressSpinner("Loading bundle metadata") @@ -45,7 +43,7 @@ func (b *Bundle) Deploy() error { defer metadataSpinner.Stop() // Check that provided oci source path is valid, and update it if it's missing the full path - source, err := CheckOCISourcePath(ctx, b.cfg.DeployOpts.Source) + source, err := CheckOCISourcePath(b.cfg.DeployOpts.Source) if err != nil { return err } @@ -58,7 +56,7 @@ func (b *Bundle) Deploy() error { } // create a new provider - provider, err := NewBundleProvider(ctx, b.cfg.DeployOpts.Source, b.tmp) + provider, err := NewBundleProvider(b.cfg.DeployOpts.Source, b.tmp) if err != nil { return err } diff --git a/src/pkg/bundle/inspect.go b/src/pkg/bundle/inspect.go index f88eddcf..6ec5c5d4 100644 --- a/src/pkg/bundle/inspect.go +++ b/src/pkg/bundle/inspect.go @@ -5,25 +5,22 @@ package bundle import ( - "context" - "github.com/defenseunicorns/uds-cli/src/config" "github.com/defenseunicorns/zarf/src/pkg/utils" ) // Inspect pulls/unpacks a bundle's metadata and shows it func (b *Bundle) Inspect() error { - ctx := context.TODO() // Check that provided oci source path is valid, and update it if it's missing the full path - source, err := CheckOCISourcePath(ctx, b.cfg.InspectOpts.Source) + source, err := CheckOCISourcePath(b.cfg.InspectOpts.Source) if err != nil { return err } b.cfg.InspectOpts.Source = source // create a new provider - provider, err := NewBundleProvider(ctx, b.cfg.InspectOpts.Source, b.tmp) + provider, err := NewBundleProvider(b.cfg.InspectOpts.Source, b.tmp) if err != nil { return err } diff --git a/src/pkg/bundle/provider.go b/src/pkg/bundle/provider.go index 50996e2d..6a5cef1d 100644 --- a/src/pkg/bundle/provider.go +++ b/src/pkg/bundle/provider.go @@ -51,9 +51,10 @@ type Provider interface { } // NewBundleProvider returns a new bundler Provider based on the source type -func NewBundleProvider(ctx context.Context, source, destination string) (Provider, error) { +func NewBundleProvider(source, destination string) (Provider, error) { + ctx := context.TODO() if helpers.IsOCIURL(source) { - op := ociProvider{ctx: ctx, src: source, dst: destination} + op := ociProvider{src: source, dst: destination} platform := ocispec.Platform{ Architecture: config.GetArch(), OS: oci.MultiOS, diff --git a/src/pkg/bundle/publish.go b/src/pkg/bundle/publish.go index f4921613..e9c97501 100644 --- a/src/pkg/bundle/publish.go +++ b/src/pkg/bundle/publish.go @@ -5,7 +5,6 @@ package bundle import ( - "context" "fmt" "os" "path/filepath" @@ -25,7 +24,7 @@ func (b *Bundle) Publish() error { // load bundle metadata into memory // todo: having the tmp dir be the provider.dst is weird - provider, err := NewBundleProvider(context.TODO(), b.cfg.PublishOpts.Source, b.tmp) + provider, err := NewBundleProvider(b.cfg.PublishOpts.Source, b.tmp) if err != nil { return err } diff --git a/src/pkg/bundle/pull.go b/src/pkg/bundle/pull.go index 3d6d8cfe..41815746 100644 --- a/src/pkg/bundle/pull.go +++ b/src/pkg/bundle/pull.go @@ -34,13 +34,13 @@ func (b *Bundle) Pull() error { } // Get validated source path - source, err := CheckOCISourcePath(ctx, b.cfg.PullOpts.Source) + source, err := CheckOCISourcePath(b.cfg.PullOpts.Source) if err != nil { return err } b.cfg.PullOpts.Source = source - provider, err := NewBundleProvider(ctx, b.cfg.PullOpts.Source, cacheDir) + provider, err := NewBundleProvider(b.cfg.PullOpts.Source, cacheDir) if err != nil { return err } @@ -125,7 +125,7 @@ func (b *Bundle) Pull() error { } // tarball the bundle - if err := format.Archive(context.TODO(), out, files); err != nil { + if err := format.Archive(ctx, out, files); err != nil { return err } diff --git a/src/pkg/bundle/remote.go b/src/pkg/bundle/remote.go index cfe53c24..b3977923 100644 --- a/src/pkg/bundle/remote.go +++ b/src/pkg/bundle/remote.go @@ -39,7 +39,6 @@ const ( ) type ociProvider struct { - ctx context.Context src string dst string *oci.OrasRemote @@ -55,11 +54,12 @@ func (op *ociProvider) getBundleManifest() (*oci.Manifest, error) { // LoadBundleMetadata loads a remote bundle's metadata func (op *ociProvider) LoadBundleMetadata() (types.PathMap, error) { + ctx := context.TODO() if err := zarfUtils.CreateDirectory(filepath.Join(op.dst, config.BlobsDir), 0700); err != nil { return nil, err } - layers, err := op.PullPaths(op.ctx, filepath.Join(op.dst, config.BlobsDir), config.BundleAlwaysPull) + layers, err := op.PullPaths(ctx, filepath.Join(op.dst, config.BlobsDir), config.BundleAlwaysPull) if err != nil { return nil, err } @@ -79,8 +79,9 @@ func (op *ociProvider) LoadBundleMetadata() (types.PathMap, error) { // CreateBundleSBOM creates a bundle-level SBOM from the underlying Zarf packages, if the Zarf package contains an SBOM func (op *ociProvider) CreateBundleSBOM(extractSBOM bool) error { + ctx := context.TODO() SBOMArtifactPathMap := make(types.PathMap) - root, err := op.FetchRoot(op.ctx) + root, err := op.FetchRoot(ctx) if err != nil { return err } @@ -96,7 +97,7 @@ func (op *ociProvider) CreateBundleSBOM(extractSBOM bool) error { if layer.Annotations[ocispec.AnnotationTitle] == config.BundleYAML { continue } - zarfManifest, err := op.OrasRemote.FetchManifest(op.ctx, layer) + zarfManifest, err := op.OrasRemote.FetchManifest(ctx, layer) if err != nil { return err } @@ -107,7 +108,7 @@ func (op *ociProvider) CreateBundleSBOM(extractSBOM bool) error { continue } // grab sboms.tar and extract - sbomBytes, err := op.OrasRemote.FetchLayer(op.ctx, sbomDesc) + sbomBytes, err := op.OrasRemote.FetchLayer(ctx, sbomDesc) if err != nil { return err } @@ -142,6 +143,7 @@ func (op *ociProvider) CreateBundleSBOM(extractSBOM bool) error { // LoadBundle loads a bundle from a remote source func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.UDSBundle, types.PathMap, error) { + ctx := context.TODO() var bundle types.UDSBundle // pull the bundle's metadata + sig loaded, err := op.LoadBundleMetadata() @@ -177,7 +179,7 @@ func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.U if err != nil { return nil, nil, err } - manifestBytes, err := op.FetchLayer(op.ctx, manifestDesc) + manifestBytes, err := op.FetchLayer(ctx, manifestDesc) if err != nil { return nil, nil, err } @@ -192,7 +194,7 @@ func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.U // go through the layers in the zarf image manifest and check if they exist in the remote for _, layer := range manifest.Layers { - ok, err := op.Repo().Blobs().Exists(op.ctx, layer) + ok, err := op.Repo().Blobs().Exists(ctx, layer) progressBar.Add(1) estimatedBytes += layer.Size if err != nil { @@ -206,13 +208,13 @@ func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.U progressBar.Successf("Verified %s package", pkg.Name) } - store, err := ocistore.NewWithContext(op.ctx, op.dst) + store, err := ocistore.NewWithContext(ctx, op.dst) if err != nil { return nil, nil, err } // grab the bundle root manifest and add it to the layers to pull - rootDesc, err := op.ResolveRoot(op.ctx) + rootDesc, err := op.ResolveRoot(ctx) if err != nil { return nil, nil, err } @@ -225,7 +227,7 @@ func (op *ociProvider) LoadBundle(opts types.BundlePullOptions, _ int) (*types.U doneSaving := make(chan error) go zarfUtils.RenderProgressBarForLocalDirWrite(op.dst, estimatedBytes, doneSaving, fmt.Sprintf("Pulling bundle: %s", bundle.Metadata.Name), fmt.Sprintf("Successfully pulled bundle: %s", bundle.Metadata.Name)) // note that in this case oras.Copy() copies using the bundle root manifest, not the packages directly - _, err = oras.Copy(op.ctx, op.Repo(), op.Repo().Reference.String(), store, op.Repo().Reference.String(), copyOpts) + _, err = oras.Copy(ctx, op.Repo(), op.Repo().Reference.String(), store, op.Repo().Reference.String(), copyOpts) doneSaving <- err <-doneSaving if err != nil { @@ -246,7 +248,8 @@ func (op *ociProvider) PublishBundle(_ types.UDSBundle, _ *oci.OrasRemote) error } // Returns the validated source path based on the provided oci source path -func getOCIValidatedSource(ctx context.Context, source string) (string, error) { +func getOCIValidatedSource(source string) (string, error) { + ctx := context.TODO() originalSource := source platform := ocispec.Platform{ @@ -317,11 +320,11 @@ func ValidateArch(arch string) error { } // CheckOCISourcePath checks that provided oci source path is valid, and updates it if it's missing the full path -func CheckOCISourcePath(ctx context.Context, source string) (string, error) { +func CheckOCISourcePath(source string) (string, error) { validTarballPath := utils.IsValidTarballPath(source) var err error if !validTarballPath { - source, err = getOCIValidatedSource(ctx, source) + source, err = getOCIValidatedSource(source) if err != nil { return "", err } diff --git a/src/pkg/bundle/remove.go b/src/pkg/bundle/remove.go index caf53cbc..2dd6d126 100644 --- a/src/pkg/bundle/remove.go +++ b/src/pkg/bundle/remove.go @@ -5,7 +5,6 @@ package bundle import ( - "context" "fmt" "strings" @@ -22,10 +21,9 @@ import ( // Remove removes packages deployed from a bundle func (b *Bundle) Remove() error { - ctx := context.TODO() // Check that provided oci source path is valid, and update it if it's missing the full path - source, err := CheckOCISourcePath(ctx, b.cfg.RemoveOpts.Source) + source, err := CheckOCISourcePath(b.cfg.RemoveOpts.Source) if err != nil { return err } @@ -38,7 +36,7 @@ func (b *Bundle) Remove() error { } // create a new provider - provider, err := NewBundleProvider(ctx, b.cfg.RemoveOpts.Source, b.tmp) + provider, err := NewBundleProvider(b.cfg.RemoveOpts.Source, b.tmp) if err != nil { return err } diff --git a/src/pkg/bundler/common.go b/src/pkg/bundler/common.go index 10a77a6f..57a73e9e 100644 --- a/src/pkg/bundler/common.go +++ b/src/pkg/bundler/common.go @@ -5,7 +5,6 @@ package bundler import ( - "context" "errors" "fmt" "strings" @@ -46,7 +45,7 @@ func manifestAnnotationsFromMetadata(metadata *types.UDSMetadata) map[string]str } // copied from: https://github.com/defenseunicorns/zarf/blob/main/src/pkg/oci/push.go -func pushManifestConfigFromMetadata(ctx context.Context, r *oci.OrasRemote, metadata *types.UDSMetadata, build *types.UDSBuildData) (ocispec.Descriptor, error) { +func pushManifestConfigFromMetadata(r *oci.OrasRemote, metadata *types.UDSMetadata, build *types.UDSBuildData) (ocispec.Descriptor, error) { annotations := map[string]string{ ocispec.AnnotationTitle: metadata.Name, ocispec.AnnotationDescription: metadata.Description, @@ -56,7 +55,7 @@ func pushManifestConfigFromMetadata(ctx context.Context, r *oci.OrasRemote, meta OCIVersion: "1.0.1", Annotations: annotations, } - manifestConfigDesc, err := utils.ToOCIRemote(ctx, manifestConfig, zoci.ZarfLayerMediaTypeBlob, r) + manifestConfigDesc, err := utils.ToOCIRemote(manifestConfig, zoci.ZarfLayerMediaTypeBlob, r) if err != nil { return ocispec.Descriptor{}, err } diff --git a/src/pkg/bundler/fetcher/fetcher.go b/src/pkg/bundler/fetcher/fetcher.go index bc447e0e..cf24efbd 100644 --- a/src/pkg/bundler/fetcher/fetcher.go +++ b/src/pkg/bundler/fetcher/fetcher.go @@ -53,7 +53,6 @@ func NewPkgFetcher(pkg types.Package, fetcherConfig Config) (Fetcher, error) { return nil, err } fetcher = &remoteFetcher{ - ctx: ctx, pkg: pkg, cfg: fetcherConfig, pkgRootManifest: pkgRootManifest, diff --git a/src/pkg/bundler/fetcher/remote.go b/src/pkg/bundler/fetcher/remote.go index f7a92972..56fffcaf 100644 --- a/src/pkg/bundler/fetcher/remote.go +++ b/src/pkg/bundler/fetcher/remote.go @@ -26,7 +26,6 @@ import ( ) type remoteFetcher struct { - ctx context.Context pkg types.Package cfg Config pkgRootManifest *oci.Manifest @@ -53,7 +52,7 @@ func (f *remoteFetcher) Fetch() ([]ocispec.Descriptor, error) { if err != nil { return nil, err } - err = utils.FetchLayerAndStore(f.ctx, layerDesc, f.remote.OrasRemote, f.cfg.Store) + err = utils.FetchLayerAndStore(layerDesc, f.remote.OrasRemote, f.cfg.Store) if err != nil { return nil, err } @@ -100,7 +99,7 @@ func (f *remoteFetcher) Fetch() ([]ocispec.Descriptor, error) { func (f *remoteFetcher) layersToLocalBundle(spinner *message.Spinner, currentPackageIter int, totalPackages int) ([]ocispec.Descriptor, error) { spinner.Updatef("Fetching %s package layer metadata (package %d of %d)", f.pkg.Name, currentPackageIter, totalPackages) // get only the layers that are required by the components - layersToCopy, err := utils.GetZarfLayers(f.ctx, *f.remote, f.pkg, f.pkgRootManifest) + layersToCopy, err := utils.GetZarfLayers(*f.remote, f.pkg, f.pkgRootManifest) if err != nil { return nil, err } @@ -116,6 +115,7 @@ func (f *remoteFetcher) layersToLocalBundle(spinner *message.Spinner, currentPac // remoteToLocal copies a remote Zarf pkg to a local OCI store func (f *remoteFetcher) remoteToLocal(layersToCopy []ocispec.Descriptor) ([]ocispec.Descriptor, error) { + ctx := context.TODO() // pull layers from remote and write to OCI artifact dir var descsToBundle []ocispec.Descriptor var layersToPull []ocispec.Descriptor @@ -126,7 +126,7 @@ func (f *remoteFetcher) remoteToLocal(layersToCopy []ocispec.Descriptor) ([]ocis continue } // check if layer already exists - if exists, _ := f.cfg.Store.Exists(f.ctx, layer); exists { + if exists, _ := f.cfg.Store.Exists(ctx, layer); exists { continue } else if cache.Exists(layer.Digest.Encoded()) { err := cache.Use(layer.Digest.Encoded(), filepath.Join(f.cfg.TmpDstDir, config.BlobsDir)) @@ -187,6 +187,7 @@ func (f *remoteFetcher) remoteToLocal(layersToCopy []ocispec.Descriptor) ([]ocis } func (f *remoteFetcher) GetPkgMetadata() (zarfTypes.ZarfPackage, error) { + ctx := context.TODO() platform := ocispec.Platform{ Architecture: config.GetArch(), OS: oci.MultiOS, @@ -200,7 +201,7 @@ func (f *remoteFetcher) GetPkgMetadata() (zarfTypes.ZarfPackage, error) { if err != nil { return zarfTypes.ZarfPackage{}, fmt.Errorf("bundler unable to create temp directory: %w", err) } - if _, err := remote.PullPackageMetadata(f.ctx, tmpDir); err != nil { + if _, err := remote.PullPackageMetadata(ctx, tmpDir); err != nil { return zarfTypes.ZarfPackage{}, err } zarfYAML := zarfTypes.ZarfPackage{} diff --git a/src/pkg/bundler/localbundle.go b/src/pkg/bundler/localbundle.go index 84734e29..5e4efc2e 100644 --- a/src/pkg/bundler/localbundle.go +++ b/src/pkg/bundler/localbundle.go @@ -103,7 +103,7 @@ func (lo *LocalBundle) create(signature []byte) error { message.HeaderInfof("🚧 Building Bundle") // push uds-bundle.yaml to OCI store - bundleYAMLDesc, err := pushBundleYAMLToStore(ctx, store, bundle) + bundleYAMLDesc, err := pushBundleYAMLToStore(store, bundle) if err != nil { return err } @@ -139,7 +139,7 @@ func (lo *LocalBundle) create(signature []byte) error { // push the bundle's signature todo: need to understand functionality and add tests if len(signature) > 0 { - signatureDesc, err := pushBundleSignature(ctx, store, signature) + signatureDesc, err := pushBundleSignature(store, signature) if err != nil { return err } @@ -169,7 +169,8 @@ func (lo *LocalBundle) create(signature []byte) error { } // pushBundleYAMLToStore pushes the uds-bundle.yaml to a provided OCI store -func pushBundleYAMLToStore(ctx context.Context, store *ocistore.Store, bundle *types.UDSBundle) (ocispec.Descriptor, error) { +func pushBundleYAMLToStore(store *ocistore.Store, bundle *types.UDSBundle) (ocispec.Descriptor, error) { + ctx := context.TODO() bundleYAMLBytes, err := goyaml.Marshal(bundle) if err != nil { return ocispec.Descriptor{}, err @@ -272,7 +273,8 @@ jobLoop: return nil } -func pushBundleSignature(ctx context.Context, store *ocistore.Store, signature []byte) (ocispec.Descriptor, error) { +func pushBundleSignature(store *ocistore.Store, signature []byte) (ocispec.Descriptor, error) { + ctx := context.TODO() signatureDesc := content.NewDescriptorFromBytes(zoci.ZarfLayerMediaTypeBlob, signature) err := store.Push(ctx, signatureDesc, bytes.NewReader(signature)) if err != nil { diff --git a/src/pkg/bundler/pusher/remote.go b/src/pkg/bundler/pusher/remote.go index 35bd36eb..1012157b 100644 --- a/src/pkg/bundler/pusher/remote.go +++ b/src/pkg/bundler/pusher/remote.go @@ -20,7 +20,6 @@ import ( // RemotePusher contains methods for pulling remote Zarf packages into a bundle type RemotePusher struct { - ctx context.Context pkg types.Package cfg Config } @@ -37,11 +36,12 @@ type Config struct { // NewPkgPusher creates a pusher object to push Zarf pkgs to a remote bundle func NewPkgPusher(pkg types.Package, cfg Config) RemotePusher { - return RemotePusher{ctx: context.TODO(), pkg: pkg, cfg: cfg} + return RemotePusher{pkg: pkg, cfg: cfg} } // Push pushes a Zarf pkg to a remote bundle func (p *RemotePusher) Push() (ocispec.Descriptor, error) { + ctx := context.TODO() zarfManifestDesc, err := p.PushManifest() if err != nil { return ocispec.Descriptor{}, err @@ -53,7 +53,7 @@ func (p *RemotePusher) Push() (ocispec.Descriptor, error) { message.Debugf("Pushed %s sub-manifest into %s: %s", url, p.cfg.RemoteDst.Repo().Reference, message.JSONValue(zarfManifestDesc)) // add package name annotations to zarf manifest - zarfYamlFile, err := p.cfg.RemoteSrc.FetchZarfYAML(p.ctx) + zarfYamlFile, err := p.cfg.RemoteSrc.FetchZarfYAML(ctx) if err != nil { return ocispec.Descriptor{}, err } @@ -76,7 +76,7 @@ func (p *RemotePusher) Push() (ocispec.Descriptor, error) { // PushManifest pushes the Zarf pkg's manifest to either a local or remote bundle func (p *RemotePusher) PushManifest() (ocispec.Descriptor, error) { var zarfManifestDesc ocispec.Descriptor - desc, err := utils.ToOCIRemote(p.ctx, p.cfg.PkgRootManifest, zoci.ZarfLayerMediaTypeBlob, p.cfg.RemoteDst.OrasRemote) + desc, err := utils.ToOCIRemote(p.cfg.PkgRootManifest, zoci.ZarfLayerMediaTypeBlob, p.cfg.RemoteDst.OrasRemote) if err != nil { return ocispec.Descriptor{}, err } @@ -88,7 +88,7 @@ func (p *RemotePusher) PushManifest() (ocispec.Descriptor, error) { func (p *RemotePusher) LayersToRemoteBundle(spinner *message.Spinner, currentPackageIter int, totalPackages int) ([]ocispec.Descriptor, error) { spinner.Updatef("Fetching %s package layer metadata (package %d of %d)", p.pkg.Name, currentPackageIter, totalPackages) // get only the layers that are required by the components - layersToCopy, err := utils.GetZarfLayers(p.ctx, p.cfg.RemoteSrc, p.pkg, p.cfg.PkgRootManifest) + layersToCopy, err := utils.GetZarfLayers(p.cfg.RemoteSrc, p.pkg, p.cfg.PkgRootManifest) if err != nil { return nil, err } @@ -103,6 +103,7 @@ func (p *RemotePusher) LayersToRemoteBundle(spinner *message.Spinner, currentPac // remoteToRemote copies a remote Zarf pkg to a remote OCI registry func (p *RemotePusher) remoteToRemote(layersToCopy []ocispec.Descriptor) error { + ctx := context.TODO() srcRef := p.cfg.RemoteSrc.Repo().Reference dstRef := p.cfg.RemoteDst.Repo().Reference // stream copy if different registry @@ -118,7 +119,7 @@ func (p *RemotePusher) remoteToRemote(layersToCopy []ocispec.Descriptor) error { } return false } - if err := oci.Copy(p.ctx, p.cfg.RemoteSrc.OrasRemote, p.cfg.RemoteDst.OrasRemote, filterLayers, config.CommonOptions.OCIConcurrency, nil); err != nil { + if err := oci.Copy(ctx, p.cfg.RemoteSrc.OrasRemote, p.cfg.RemoteDst.OrasRemote, filterLayers, config.CommonOptions.OCIConcurrency, nil); err != nil { return err } } else { @@ -131,8 +132,8 @@ func (p *RemotePusher) remoteToRemote(layersToCopy []ocispec.Descriptor) error { continue } spinner.Updatef("Mounting %s", layer.Digest.Encoded()) - if err := p.cfg.RemoteDst.Repo().Mount(p.ctx, layer, srcRef.Repository, func() (io.ReadCloser, error) { - return p.cfg.RemoteSrc.Repo().Fetch(p.ctx, layer) + if err := p.cfg.RemoteDst.Repo().Mount(ctx, layer, srcRef.Repository, func() (io.ReadCloser, error) { + return p.cfg.RemoteSrc.Repo().Fetch(ctx, layer) }); err != nil { return err } diff --git a/src/pkg/bundler/remotebundle.go b/src/pkg/bundler/remotebundle.go index f3955f45..e9f63dfd 100644 --- a/src/pkg/bundler/remotebundle.go +++ b/src/pkg/bundler/remotebundle.go @@ -31,7 +31,6 @@ type RemoteBundle struct { bundle *types.UDSBundle tmpDstDir string output string - ctx context.Context } // NewRemoteBundle creates a new remote bundle @@ -40,12 +39,13 @@ func NewRemoteBundle(opts *RemoteBundleOpts) *RemoteBundle { bundle: opts.Bundle, tmpDstDir: opts.TmpDstDir, output: opts.Output, - ctx: context.TODO(), } } // create creates the bundle in a remote OCI registry publishes w/ optional signature to the remote repository. func (r *RemoteBundle) create(signature []byte) error { + ctx := context.TODO() + // set the bundle remote's reference from metadata r.output = utils.EnsureOCIPrefix(r.output) ref, err := referenceFromMetadata(r.output, &r.bundle.Metadata) @@ -84,7 +84,7 @@ func (r *RemoteBundle) create(signature []byte) error { return err } pusherConfig.RemoteSrc = *src - pkgRootManifest, err := src.FetchRoot(r.ctx) + pkgRootManifest, err := src.FetchRoot(ctx) if err != nil { return err } @@ -104,7 +104,7 @@ func (r *RemoteBundle) create(signature []byte) error { if err != nil { return err } - bundleYamlDesc, err := bundleRemote.PushLayer(r.ctx, bundleYamlBytes, zoci.ZarfLayerMediaTypeBlob) + bundleYamlDesc, err := bundleRemote.PushLayer(ctx, bundleYamlBytes, zoci.ZarfLayerMediaTypeBlob) if err != nil { return err } @@ -117,7 +117,7 @@ func (r *RemoteBundle) create(signature []byte) error { // push the bundle's signature if len(signature) > 0 { - bundleYamlSigDesc, err := bundleRemote.PushLayer(r.ctx, signature, zoci.ZarfLayerMediaTypeBlob) + bundleYamlSigDesc, err := bundleRemote.PushLayer(ctx, signature, zoci.ZarfLayerMediaTypeBlob) if err != nil { return err } @@ -129,7 +129,7 @@ func (r *RemoteBundle) create(signature []byte) error { } // push the bundle manifest config - configDesc, err := pushManifestConfigFromMetadata(r.ctx, bundleRemote.OrasRemote, &bundle.Metadata, &bundle.Build) + configDesc, err := pushManifestConfigFromMetadata(bundleRemote.OrasRemote, &bundle.Metadata, &bundle.Build) if err != nil { return err } @@ -146,7 +146,7 @@ func (r *RemoteBundle) create(signature []byte) error { rootManifest.Config = configDesc rootManifest.SchemaVersion = 2 rootManifest.Annotations = manifestAnnotationsFromMetadata(&bundle.Metadata) // maps to registry UI - rootManifestDesc, err := utils.ToOCIRemote(r.ctx, rootManifest, ocispec.MediaTypeImageManifest, bundleRemote.OrasRemote) + rootManifestDesc, err := utils.ToOCIRemote(rootManifest, ocispec.MediaTypeImageManifest, bundleRemote.OrasRemote) if err != nil { return err } diff --git a/src/pkg/sources/new.go b/src/pkg/sources/new.go index 1a3cd677..f65730a6 100644 --- a/src/pkg/sources/new.go +++ b/src/pkg/sources/new.go @@ -5,7 +5,6 @@ package sources import ( - "context" "strings" "github.com/defenseunicorns/uds-cli/src/config" @@ -42,7 +41,6 @@ func New(pkgLocation string, pkgName string, opts zarfTypes.ZarfPackageOptions, PkgManifestSHA: sha, TmpDir: opts.PackageSource, Remote: remote.OrasRemote, - ctx: context.TODO(), } } return source, nil diff --git a/src/pkg/sources/remote.go b/src/pkg/sources/remote.go index f45dc0ce..5efdc76d 100644 --- a/src/pkg/sources/remote.go +++ b/src/pkg/sources/remote.go @@ -35,7 +35,6 @@ type RemoteBundle struct { TmpDir string Remote *oci.OrasRemote isPartial bool - ctx context.Context } // LoadPackage loads a Zarf package from a remote bundle @@ -82,7 +81,8 @@ func (r *RemoteBundle) LoadPackage(dst *layout.PackagePaths, unarchiveAll bool) // LoadPackageMetadata loads a Zarf package's metadata from a remote bundle func (r *RemoteBundle) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ bool) (err error) { - root, err := r.Remote.FetchRoot(r.ctx) + ctx := context.TODO() + root, err := r.Remote.FetchRoot(ctx) if err != nil { return err } @@ -92,7 +92,7 @@ func (r *RemoteBundle) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ b } // look at Zarf pkg manifest, grab zarf.yaml desc and download it - pkgManifest, err := r.Remote.FetchManifest(r.ctx, pkgManifestDesc) + pkgManifest, err := r.Remote.FetchManifest(ctx, pkgManifestDesc) if err != nil { return err } @@ -104,7 +104,7 @@ func (r *RemoteBundle) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ b break } } - zarfYAMLBytes, err := r.Remote.FetchLayer(r.ctx, zarfYAMLDesc) + zarfYAMLBytes, err := r.Remote.FetchLayer(ctx, zarfYAMLDesc) if err != nil { return err } @@ -121,7 +121,7 @@ func (r *RemoteBundle) LoadPackageMetadata(dst *layout.PackagePaths, _ bool, _ b var checksumLayer ocispec.Descriptor for _, layer := range pkgManifest.Layers { if layer.Annotations[ocispec.AnnotationTitle] == config.ChecksumsTxt { - checksumBytes, err := r.Remote.FetchLayer(r.ctx, layer) + checksumBytes, err := r.Remote.FetchLayer(ctx, layer) if err != nil { return err } @@ -147,7 +147,8 @@ func (r *RemoteBundle) Collect(_ string) (string, error) { // downloadPkgFromRemoteBundle downloads a Zarf package from a remote bundle func (r *RemoteBundle) downloadPkgFromRemoteBundle() ([]ocispec.Descriptor, error) { - rootManifest, err := r.Remote.FetchRoot(r.ctx) + ctx := context.TODO() + rootManifest, err := r.Remote.FetchRoot(ctx) if err != nil { return nil, err } @@ -158,7 +159,7 @@ func (r *RemoteBundle) downloadPkgFromRemoteBundle() ([]ocispec.Descriptor, erro } // hack Zarf media type so that FetchManifest works pkgManifestDesc.MediaType = zoci.ZarfLayerMediaTypeBlob - pkgManifest, err := r.Remote.FetchManifest(r.ctx, pkgManifestDesc) + pkgManifest, err := r.Remote.FetchManifest(ctx, pkgManifestDesc) if err != nil || pkgManifest == nil { return nil, err } @@ -171,7 +172,7 @@ func (r *RemoteBundle) downloadPkgFromRemoteBundle() ([]ocispec.Descriptor, erro layersInBundle := []ocispec.Descriptor{pkgManifestDesc} for _, layer := range pkgManifest.Layers { - ok, err := r.Remote.Repo().Blobs().Exists(context.TODO(), layer) + ok, err := r.Remote.Repo().Blobs().Exists(ctx, layer) if err != nil { return nil, err } @@ -204,7 +205,7 @@ func (r *RemoteBundle) downloadPkgFromRemoteBundle() ([]ocispec.Descriptor, erro copyOpts := utils.CreateCopyOpts(layersToPull, config.CommonOptions.OCIConcurrency) doneSaving := make(chan error) go zarfUtils.RenderProgressBarForLocalDirWrite(r.TmpDir, estimatedBytes, doneSaving, fmt.Sprintf("Pulling bundled Zarf pkg: %s", r.PkgName), fmt.Sprintf("Successfully pulled package: %s", r.PkgName)) - _, err = oras.Copy(context.TODO(), r.Remote.Repo(), r.Remote.Repo().Reference.String(), store, "", copyOpts) + _, err = oras.Copy(ctx, r.Remote.Repo(), r.Remote.Repo().Reference.String(), store, "", copyOpts) doneSaving <- err <-doneSaving if err != nil { diff --git a/src/pkg/utils/oci.go b/src/pkg/utils/oci.go index a0cae61d..527fdda9 100644 --- a/src/pkg/utils/oci.go +++ b/src/pkg/utils/oci.go @@ -26,7 +26,8 @@ import ( ) // FetchLayerAndStore fetches a remote layer and copies it to a local store -func FetchLayerAndStore(ctx context.Context, layerDesc ocispec.Descriptor, remoteRepo *oci.OrasRemote, localStore *ocistore.Store) error { +func FetchLayerAndStore(layerDesc ocispec.Descriptor, remoteRepo *oci.OrasRemote, localStore *ocistore.Store) error { + ctx := context.TODO() layerBytes, err := remoteRepo.FetchLayer(ctx, layerDesc) if err != nil { return err @@ -53,7 +54,8 @@ func ToOCIStore(t any, mediaType string, store *ocistore.Store) (ocispec.Descrip } // ToOCIRemote takes an arbitrary type, typically a struct, marshals it into JSON and store it in a remote OCI store -func ToOCIRemote(ctx context.Context, t any, mediaType string, remote *oci.OrasRemote) (*ocispec.Descriptor, error) { +func ToOCIRemote(t any, mediaType string, remote *oci.OrasRemote) (*ocispec.Descriptor, error) { + ctx := context.TODO() b, err := json.Marshal(t) if err != nil { return &ocispec.Descriptor{}, err @@ -63,8 +65,7 @@ func ToOCIRemote(ctx context.Context, t any, mediaType string, remote *oci.OrasR // if image manifest media type, push to Manifests(), otherwise normal pushLayer() if mediaType == ocispec.MediaTypeImageManifest { descriptorFromBytes := content.NewDescriptorFromBytes(ocispec.MediaTypeImageManifest, b) - layerDesc = &descriptorFromBytes - if err := remote.Repo().Manifests().PushReference(context.TODO(), *layerDesc, bytes.NewReader(b), remote.Repo().Reference.String()); err != nil { + if err := remote.Repo().Manifests().PushReference(ctx, descriptorFromBytes, bytes.NewReader(b), remote.Repo().Reference.String()); err != nil { return &ocispec.Descriptor{}, fmt.Errorf("failed to push manifest: %w", err) } } else { @@ -220,8 +221,9 @@ func UpdateIndex(index *ocispec.Index, remote *oci.OrasRemote, bundle *types.UDS // GetIndex gets the OCI index from a remote repository if the index exists, otherwise returns a func GetIndex(remote *oci.OrasRemote, ref string) (*ocispec.Index, error) { + ctx := context.TODO() var index *ocispec.Index - existingRootDesc, err := remote.Repo().Resolve(context.TODO(), ref) + existingRootDesc, err := remote.Repo().Resolve(ctx, ref) if err != nil { // ErrNotFound indicates that the repo hasn't been created yet, expected for brand new repos in a registry // if the err isn't of type ErrNotFound, it's a real error so return it @@ -231,7 +233,7 @@ func GetIndex(remote *oci.OrasRemote, ref string) (*ocispec.Index, error) { } // if an index exists, save it so we can update it after pushing the bundle's root manifest if existingRootDesc.MediaType == ocispec.MediaTypeImageIndex { - rc, err := remote.Repo().Fetch(context.TODO(), existingRootDesc) + rc, err := remote.Repo().Fetch(ctx, existingRootDesc) if err != nil { return nil, err } @@ -257,7 +259,8 @@ func EnsureOCIPrefix(source string) string { } // GetZarfLayers grabs the necessary Zarf pkg layers from a remote OCI registry -func GetZarfLayers(ctx context.Context, remote zoci.Remote, pkg types.Package, pkgRootManifest *oci.Manifest) ([]ocispec.Descriptor, error) { +func GetZarfLayers(remote zoci.Remote, pkg types.Package, pkgRootManifest *oci.Manifest) ([]ocispec.Descriptor, error) { + ctx := context.TODO() layersFromComponents, err := remote.LayersFromRequestedComponents(ctx, pkg.OptionalComponents) if err != nil { return nil, err From 8c4ac4ab2f5676a99c5dd5e620d878452a886718 Mon Sep 17 00:00:00 2001 From: Darcy Cleaver Date: Fri, 15 Mar 2024 13:31:06 -0600 Subject: [PATCH 7/7] revert removing the setting of layerDesc since it gets used as part of the success message --- src/pkg/utils/oci.go | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pkg/utils/oci.go b/src/pkg/utils/oci.go index 527fdda9..b8fff8bd 100644 --- a/src/pkg/utils/oci.go +++ b/src/pkg/utils/oci.go @@ -65,6 +65,7 @@ func ToOCIRemote(t any, mediaType string, remote *oci.OrasRemote) (*ocispec.Desc // if image manifest media type, push to Manifests(), otherwise normal pushLayer() if mediaType == ocispec.MediaTypeImageManifest { descriptorFromBytes := content.NewDescriptorFromBytes(ocispec.MediaTypeImageManifest, b) + layerDesc = &descriptorFromBytes if err := remote.Repo().Manifests().PushReference(ctx, descriptorFromBytes, bytes.NewReader(b), remote.Repo().Reference.String()); err != nil { return &ocispec.Descriptor{}, fmt.Errorf("failed to push manifest: %w", err) }