diff --git a/go.mod b/go.mod index aebabfdbd4..5f2ceb4110 100644 --- a/go.mod +++ b/go.mod @@ -230,3 +230,5 @@ require ( replace github.com/containers/common => github.com/mtrmac/common v0.8.2-0.20241015204126-78c8c087610a replace github.com/containers/image/v5 => github.com/mtrmac/image/v5 v5.0.0-20241016195457-6045533dd9a2 + +replace github.com/containers/storage => github.com/mtrmac/storage v0.0.0-20241017234338-b057acd7a73d diff --git a/go.sum b/go.sum index a6d39f9c14..8bf498b133 100644 --- a/go.sum +++ b/go.sum @@ -93,8 +93,6 @@ github.com/containers/ocicrypt v1.2.0 h1:X14EgRK3xNFvJEfI5O4Qn4T3E25ANudSOZz/sir github.com/containers/ocicrypt v1.2.0/go.mod h1:ZNviigQajtdlxIZGibvblVuIFBKIuUI2M0QM12SD31U= github.com/containers/psgo v1.9.0 h1:eJ74jzSaCHnWt26OlKZROSyUyRcGDf+gYBdXnxrMW4g= github.com/containers/psgo v1.9.0/go.mod h1:0YoluUm43Mz2UnBIh1P+6V6NWcbpTL5uRtXyOcH0B5A= -github.com/containers/storage v1.55.1-0.20241016074337-b417e8d6376d h1:pEpS4UWW++IMwvWHLOVRYPofnRWFora35fHx/XwM18c= -github.com/containers/storage v1.55.1-0.20241016074337-b417e8d6376d/go.mod h1:1Yv4k1PJCLjKLhQ5N4TdH7+5ntDjRcy/dHcZQKRQ4fg= github.com/containers/winquit v1.1.0 h1:jArun04BNDQvt2W0Y78kh9TazN2EIEMG5Im6/JY7+pE= github.com/containers/winquit v1.1.0/go.mod h1:PsPeZlnbkmGGIToMPHF1zhWjBUkd8aHjMOr/vFcPxw8= github.com/coreos/go-oidc/v3 v3.11.0 h1:Ia3MxdwpSw702YW0xgfmP1GVCMA9aEFWu12XUZ3/OtI= @@ -382,6 +380,8 @@ github.com/mtrmac/common v0.8.2-0.20241015204126-78c8c087610a h1:xpe1z25rOIfC8CD github.com/mtrmac/common v0.8.2-0.20241015204126-78c8c087610a/go.mod h1:GRT29AbW4CdqEWP/jSxHyUvV5fprOzsCdhsFhqJiU4s= github.com/mtrmac/image/v5 v5.0.0-20241016195457-6045533dd9a2 h1:jv0/6AabBWh1KPQPTD9nefFwVvimQA0K+phljh8mNU8= github.com/mtrmac/image/v5 v5.0.0-20241016195457-6045533dd9a2/go.mod h1:Ulwf/jQO4757C/uOJyNiZ10dRiXRwVnyhF9wYFno3GQ= +github.com/mtrmac/storage v0.0.0-20241017234338-b057acd7a73d h1:/80cSp1xPePUfa7DheEQUAESxKcUD1sspiaG6l/5jPA= +github.com/mtrmac/storage v0.0.0-20241017234338-b057acd7a73d/go.mod h1:iq56tOFXnj8kA8DAytN28fhUm77eKLWrashQrImaqBs= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= diff --git a/vendor/github.com/containers/storage/.cirrus.yml b/vendor/github.com/containers/storage/.cirrus.yml index 3e5fa6ad4f..0fed08c3b2 100644 --- a/vendor/github.com/containers/storage/.cirrus.yml +++ b/vendor/github.com/containers/storage/.cirrus.yml @@ -180,6 +180,13 @@ gofix_task: build_script: go fix ./... test_script: git diff --exit-code +codespell_task: + alias: codespell + container: + image: python + build_script: pip install codespell + test_script: codespell + # Status aggregator for all tests. This task simply ensures a defined # set of tasks all passed, and allows confirming that based on the status @@ -197,6 +204,7 @@ success_task: - vendor - cross - gofix + - codespell container: image: golang:1.21 clone_script: 'mkdir -p "$CIRRUS_WORKING_DIR"' # Source code not needed diff --git a/vendor/github.com/containers/storage/.codespellrc b/vendor/github.com/containers/storage/.codespellrc new file mode 100644 index 0000000000..2af969196e --- /dev/null +++ b/vendor/github.com/containers/storage/.codespellrc @@ -0,0 +1,3 @@ +[codespell] +skip = ./.git,./vendor,./tests/tools/vendor,AUTHORS +ignore-words-list = afile,flate,prevend,Plack,worl diff --git a/vendor/github.com/containers/storage/Makefile b/vendor/github.com/containers/storage/Makefile index a619694fd0..64c01b67f8 100644 --- a/vendor/github.com/containers/storage/Makefile +++ b/vendor/github.com/containers/storage/Makefile @@ -46,7 +46,7 @@ containers-storage: ## build using gc on the host $(GO) build -compiler gc $(BUILDFLAGS) ./cmd/containers-storage codespell: - codespell -S Makefile,build,buildah,buildah.spec,imgtype,copy,AUTHORS,bin,vendor,.git,go.sum,CHANGELOG.md,changelog.txt,seccomp.json,.cirrus.yml,"*.xz,*.gz,*.tar,*.tgz,*ico,*.png,*.1,*.5,*.orig,*.rej" -L plack,worl,flate,uint,iff,od,ERRO -w + codespell binary local-binary: containers-storage diff --git a/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go b/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go index 1db03b2e7a..47742b05d8 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/cache_linux.go @@ -274,7 +274,7 @@ func (c *layersCache) load() error { var newLayers []*layer for _, r := range allLayers { // The layer is present in the store and it is already loaded. Attempt to - // re-use it if mmap'ed. + // reuse it if mmap'ed. if l, found := loadedLayers[r.ID]; found { // If the layer is not marked for re-load, move it to newLayers. if !l.reloadWithMmap { diff --git a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go index a64787838f..a11cfb5800 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go @@ -150,61 +150,89 @@ func GetDiffer(ctx context.Context, store storage.Store, blobDigest digest.Diges pullOptions := store.PullOptions() if !parseBooleanPullOption(pullOptions, "enable_partial_images", true) { + // If convertImages is set, the two options disagree whether fallback is permissible. + // Right now, we enable it, but that’s not a promise; rather, such a configuration should ideally be rejected. return nil, newErrFallbackToOrdinaryLayerDownload(errors.New("partial images are disabled")) } - - zstdChunkedTOCDigestString, hasZstdChunkedTOC := annotations[internal.ManifestChecksumKey] - estargzTOCDigestString, hasEstargzTOC := annotations[estargz.TOCJSONDigestAnnotation] - - if hasZstdChunkedTOC && hasEstargzTOC { - return nil, errors.New("both zstd:chunked and eStargz TOC found") - } - + // convertImages also serves as a “must not fallback to non-partial pull” option (?!) convertImages := parseBooleanPullOption(pullOptions, "convert_images", false) - if !hasZstdChunkedTOC && !hasEstargzTOC && !convertImages { - return nil, newErrFallbackToOrdinaryLayerDownload(errors.New("no TOC found and convert_images is not configured")) + graphDriver, err := store.GraphDriver() + if err != nil { + return nil, err } - - var err error - var differ graphdriver.Differ - // At this point one of hasZstdChunkedTOC, hasEstargzTOC or convertImages is true. - if hasZstdChunkedTOC { - zstdChunkedTOCDigest, err2 := digest.Parse(zstdChunkedTOCDigestString) - if err2 != nil { - return nil, err2 - } - differ, err = makeZstdChunkedDiffer(store, blobSize, zstdChunkedTOCDigest, annotations, iss, pullOptions) - if err == nil { - logrus.Debugf("Created zstd:chunked differ for blob %q", blobDigest) - return differ, err + if _, partialSupported := graphDriver.(graphdriver.DriverWithDiffer); !partialSupported { + if convertImages { + return nil, fmt.Errorf("graph driver %s does not support partial pull but convert_images requires that", graphDriver.String()) } - } else if hasEstargzTOC { - estargzTOCDigest, err2 := digest.Parse(estargzTOCDigestString) - if err2 != nil { + return nil, newErrFallbackToOrdinaryLayerDownload(fmt.Errorf("graph driver %s does not support partial pull", graphDriver.String())) + } + + differ, canFallback, err := getRealDiffer(store, blobDigest, blobSize, annotations, iss, pullOptions) + if err != nil { + if !canFallback { return nil, err } - differ, err = makeEstargzChunkedDiffer(store, blobSize, estargzTOCDigest, iss, pullOptions) - if err == nil { - logrus.Debugf("Created eStargz differ for blob %q", blobDigest) - return differ, err + // If convert_images is enabled, always attempt to convert it instead of returning an error or falling back to a different method. + if convertImages { + logrus.Debugf("Created differ to convert blob %q", blobDigest) + return makeConvertFromRawDiffer(store, blobDigest, blobSize, iss, pullOptions) } - } - // If convert_images is enabled, always attempt to convert it instead of returning an error or falling back to a different method. - if convertImages { - logrus.Debugf("Created differ to convert blob %q", blobDigest) - return makeConvertFromRawDiffer(store, blobDigest, blobSize, iss, pullOptions) + return nil, newErrFallbackToOrdinaryLayerDownload(err) } - logrus.Debugf("Could not create differ for blob %q: %v", blobDigest, err) + return differ, nil +} - // If the error is a bad request to the server, then signal to the caller that it can try a different method. This can be done - // only when convert_images is disabled. - var badRequestErr ErrBadRequest - if errors.As(err, &badRequestErr) { - err = newErrFallbackToOrdinaryLayerDownload(err) +// getProperDiffer is an implementation detail of GetDiffer. +// It returns a “proper” differ (not a convert_images one) if possible. +// On error, the second parameter is true if a fallback to an alternative (either the makeConverToRaw differ, or a non-partial pull) +// is permissible. +func getRealDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, annotations map[string]string, iss ImageSourceSeekable, pullOptions map[string]string) (graphdriver.Differ, bool, error) { + zstdChunkedTOCDigestString, hasZstdChunkedTOC := annotations[internal.ManifestChecksumKey] + estargzTOCDigestString, hasEstargzTOC := annotations[estargz.TOCJSONDigestAnnotation] + + switch { + case hasZstdChunkedTOC && hasEstargzTOC: + return nil, false, errors.New("both zstd:chunked and eStargz TOC found") + + case hasZstdChunkedTOC: + zstdChunkedTOCDigest, err := digest.Parse(zstdChunkedTOCDigestString) + if err != nil { + return nil, false, err + } + differ, err := makeZstdChunkedDiffer(store, blobSize, zstdChunkedTOCDigest, annotations, iss, pullOptions) + if err != nil { + logrus.Debugf("Could not create zstd:chunked differ for blob %q: %v", blobDigest, err) + // If the error is a bad request to the server, then signal to the caller that it can try a different method. + var badRequestErr ErrBadRequest + return nil, errors.As(err, &badRequestErr), err + } + logrus.Debugf("Created zstd:chunked differ for blob %q", blobDigest) + return differ, false, nil + + case hasEstargzTOC: + estargzTOCDigest, err := digest.Parse(estargzTOCDigestString) + if err != nil { + return nil, false, err + } + differ, err := makeEstargzChunkedDiffer(store, blobSize, estargzTOCDigest, iss, pullOptions) + if err != nil { + logrus.Debugf("Could not create estargz differ for blob %q: %v", blobDigest, err) + // If the error is a bad request to the server, then signal to the caller that it can try a different method. + var badRequestErr ErrBadRequest + return nil, errors.As(err, &badRequestErr), err + } + logrus.Debugf("Created eStargz differ for blob %q", blobDigest) + return differ, false, nil + + default: // no TOC + convertImages := parseBooleanPullOption(pullOptions, "convert_images", false) + if !convertImages { + return nil, true, errors.New("no TOC found and convert_images is not configured") + } + return nil, true, errors.New("no TOC found") } - return nil, err } func makeConvertFromRawDiffer(store storage.Store, blobDigest digest.Digest, blobSize int64, iss ImageSourceSeekable, pullOptions map[string]string) (*chunkedDiffer, error) { diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools.go b/vendor/github.com/containers/storage/pkg/idtools/idtools.go index dc963481a3..248594e935 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/idtools.go +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools.go @@ -367,7 +367,7 @@ func checkChownErr(err error, name string, uid, gid int) error { return err } -// Stat contains file states that can be overriden with ContainersOverrideXattr. +// Stat contains file states that can be overridden with ContainersOverrideXattr. type Stat struct { IDs IDPair Mode os.FileMode diff --git a/vendor/modules.txt b/vendor/modules.txt index 06ae69650d..b00f378d6d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -353,7 +353,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.55.1-0.20241016074337-b417e8d6376d +# github.com/containers/storage v1.55.1-0.20241016074337-b417e8d6376d => github.com/mtrmac/storage v0.0.0-20241017234338-b057acd7a73d ## explicit; go 1.22.0 github.com/containers/storage github.com/containers/storage/drivers @@ -1386,3 +1386,4 @@ tags.cncf.io/container-device-interface/pkg/parser tags.cncf.io/container-device-interface/specs-go # github.com/containers/common => github.com/mtrmac/common v0.8.2-0.20241015204126-78c8c087610a # github.com/containers/image/v5 => github.com/mtrmac/image/v5 v5.0.0-20241016195457-6045533dd9a2 +# github.com/containers/storage => github.com/mtrmac/storage v0.0.0-20241017234338-b057acd7a73d