From 9eed9c58b250a27e1dfd639cc6cc9706107b50c3 Mon Sep 17 00:00:00 2001 From: Andrei Aaron Date: Wed, 22 Nov 2023 20:00:52 +0000 Subject: [PATCH] fix(digests): do not mandate sha256 as the only algorithm used for hashing blobs Signed-off-by: Andrei Aaron --- pkg/api/controller_test.go | 239 +++++++++++++++++++++++++++ pkg/storage/common/common.go | 53 +++--- pkg/storage/gc/gc.go | 11 +- pkg/storage/gc/gc_internal_test.go | 4 +- pkg/storage/imagestore/imagestore.go | 101 +++++++---- pkg/storage/local/local_test.go | 9 +- pkg/storage/s3/s3_test.go | 54 +++--- pkg/storage/scrub.go | 2 +- pkg/storage/types/types.go | 2 +- pkg/test/image-utils/images.go | 57 +++++-- pkg/test/image-utils/multiarch.go | 43 ++++- pkg/test/image-utils/upload.go | 24 ++- pkg/test/image-utils/write.go | 10 +- pkg/test/mocks/image_store_mock.go | 6 +- 14 files changed, 482 insertions(+), 133 deletions(-) diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index a5a849ae32..68a6b1f2b0 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -10533,6 +10533,245 @@ func RunAuthorizationTests(t *testing.T, client *resty.Client, baseURL, user str }) } +func TestSupportedDigestAlgorithms(t *testing.T) { + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + + conf := config.New() + conf.HTTP.Port = port + + dir := t.TempDir() + + ctlr := api.NewController(conf) + ctlr.Config.Storage.RootDirectory = dir + ctlr.Config.Storage.Dedupe = false + ctlr.Config.Storage.GC = false + + cm := test.NewControllerManager(ctlr) + cm.StartAndWait(port) + defer cm.StopServer() + + Convey("Test SHA512 single-arch image", t, func() { + image := CreateImageWithDigestAlgorithm(godigest.SHA512). + RandomLayers(1, 10).DefaultConfig().Build() + + name := "algo-sha256" + tag := "singlearch" + + err := UploadImage(image, baseURL, name, tag) + So(err, ShouldBeNil) + + client := resty.New() + + // The server picks canonical digests when tags are pushed + // See https://github.com/opencontainers/distribution-spec/issues/494 + // It would be nice to be able to push tags with other digest algorithms and verify those are returned + // but there is no way to specify a client preference + // so all we can do is verify the correct algorithm is returned + + expectedDigestStr := image.DigestForAlgorithm(godigest.Canonical).String() + + headResponse, err := client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag)) + So(err, ShouldBeNil) + So(headResponse, ShouldNotBeNil) + So(headResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr := headResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + + getResponse, err := client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag)) + So(err, ShouldBeNil) + So(getResponse, ShouldNotBeNil) + So(getResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + + getResponse, err = client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr)) + So(err, ShouldBeNil) + So(getResponse, ShouldNotBeNil) + So(getResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + + getResponse, err = client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr)) + So(err, ShouldBeNil) + So(getResponse, ShouldNotBeNil) + So(getResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + }) + + Convey("Test SHA384 single-arch image", t, func() { + image := CreateImageWithDigestAlgorithm(godigest.SHA384). + RandomLayers(1, 10).DefaultConfig().Build() + + name := "algo-sha384" + tag := "singlearch" + + err := UploadImage(image, baseURL, name, tag) + So(err, ShouldBeNil) + + client := resty.New() + + // The server picks canonical digests when tags are pushed + // See https://github.com/opencontainers/distribution-spec/issues/494 + // It would be nice to be able to push tags with other digest algorithms and verify those are returned + // but there is no way to specify a client preference + // so all we can do is verify the correct algorithm is returned + + expectedDigestStr := image.DigestForAlgorithm(godigest.Canonical).String() + + headResponse, err := client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag)) + So(err, ShouldBeNil) + So(headResponse, ShouldNotBeNil) + So(headResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr := headResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + + getResponse, err := client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag)) + So(err, ShouldBeNil) + So(getResponse, ShouldNotBeNil) + So(getResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + + getResponse, err = client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr)) + So(err, ShouldBeNil) + So(getResponse, ShouldNotBeNil) + So(getResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + + getResponse, err = client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr)) + So(err, ShouldBeNil) + So(getResponse, ShouldNotBeNil) + So(getResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + }) + + Convey("Test SHA512 multi-arch image", t, func() { + subImage1 := CreateImageWithDigestAlgorithm(godigest.SHA512).RandomLayers(1, 10). + DefaultConfig().Build() + subImage2 := CreateImageWithDigestAlgorithm(godigest.SHA512).RandomLayers(1, 10). + DefaultConfig().Build() + multiarch := CreateMultiarchWithDigestAlgorithm(godigest.SHA512). + Images([]Image{subImage1, subImage2}).Build() + + name := "algo-sha256" + tag := "multiarch" + + err := UploadMultiarchImage(multiarch, baseURL, name, tag) + So(err, ShouldBeNil) + + client := resty.New() + + // The server picks canonical digests when tags are pushed + // See https://github.com/opencontainers/distribution-spec/issues/494 + // It would be nice to be able to push tags with other digest algorithms and verify those are returned + // but there is no way to specify a client preference + // so all we can do is verify the correct algorithm is returned + + expectedDigestStr := multiarch.DigestForAlgorithm(godigest.Canonical).String() + + headResponse, err := client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag)) + So(err, ShouldBeNil) + So(headResponse, ShouldNotBeNil) + So(headResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr := headResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + + getResponse, err := client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag)) + So(err, ShouldBeNil) + So(getResponse, ShouldNotBeNil) + So(getResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + + getResponse, err = client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr)) + So(err, ShouldBeNil) + So(getResponse, ShouldNotBeNil) + So(getResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + + getResponse, err = client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr)) + So(err, ShouldBeNil) + So(getResponse, ShouldNotBeNil) + So(getResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + }) + + Convey("Test SHA384 multi-arch image", t, func() { + subImage1 := CreateImageWithDigestAlgorithm(godigest.SHA384).RandomLayers(1, 10). + DefaultConfig().Build() + subImage2 := CreateImageWithDigestAlgorithm(godigest.SHA384).RandomLayers(1, 10). + DefaultConfig().Build() + multiarch := CreateMultiarchWithDigestAlgorithm(godigest.SHA384). + Images([]Image{subImage1, subImage2}).Build() + + name := "algo-sha384" + tag := "multiarch" + + err := UploadMultiarchImage(multiarch, baseURL, name, tag) + So(err, ShouldBeNil) + + client := resty.New() + + // The server picks canonical digests when tags are pushed + // See https://github.com/opencontainers/distribution-spec/issues/494 + // It would be nice to be able to push tags with other digest algorithms and verify those are returned + // but there is no way to specify a client preference + // so all we can do is verify the correct algorithm is returned + + expectedDigestStr := multiarch.DigestForAlgorithm(godigest.Canonical).String() + + headResponse, err := client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag)) + So(err, ShouldBeNil) + So(headResponse, ShouldNotBeNil) + So(headResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr := headResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + + getResponse, err := client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, tag)) + So(err, ShouldBeNil) + So(getResponse, ShouldNotBeNil) + So(getResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + + getResponse, err = client.R().Get(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr)) + So(err, ShouldBeNil) + So(getResponse, ShouldNotBeNil) + So(getResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + + getResponse, err = client.R().Head(fmt.Sprintf("%s/v2/%s/manifests/%s", baseURL, name, canonicalDigestStr)) + So(err, ShouldBeNil) + So(getResponse, ShouldNotBeNil) + So(getResponse.StatusCode(), ShouldEqual, http.StatusOK) + + canonicalDigestStr = getResponse.Header().Get("Docker-Content-Digest") + So(canonicalDigestStr, ShouldEqual, expectedDigestStr) + }) +} + func getEmptyImageConfig() ([]byte, godigest.Digest) { config := ispec.Image{} diff --git a/pkg/storage/common/common.go b/pkg/storage/common/common.go index 0795b649a5..f104a4bb44 100644 --- a/pkg/storage/common/common.go +++ b/pkg/storage/common/common.go @@ -64,19 +64,19 @@ func GetManifestDescByReference(index ispec.Index, reference string) (ispec.Desc func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaType string, body []byte, log zlog.Logger, -) (godigest.Digest, error) { +) error { // validate the manifest if !IsSupportedMediaType(mediaType) { log.Debug().Interface("actual", mediaType). Msg("bad manifest media type") - return "", zerr.ErrBadManifest + return zerr.ErrBadManifest } if len(body) == 0 { log.Debug().Int("len", len(body)).Msg("invalid body length") - return "", zerr.ErrBadManifest + return zerr.ErrBadManifest } switch mediaType { @@ -87,13 +87,13 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy if err := ValidateManifestSchema(body); err != nil { log.Error().Err(err).Msg("OCIv1 image manifest schema validation failed") - return "", zerr.NewError(zerr.ErrBadManifest).AddDetail("jsonSchemaValidation", err.Error()) + return zerr.NewError(zerr.ErrBadManifest).AddDetail("jsonSchemaValidation", err.Error()) } if err := json.Unmarshal(body, &manifest); err != nil { log.Error().Err(err).Msg("unable to unmarshal JSON") - return "", zerr.ErrBadManifest + return zerr.ErrBadManifest } // validate blobs only for known media types @@ -104,7 +104,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy if !ok || err != nil { log.Error().Err(err).Str("digest", manifest.Config.Digest.String()).Msg("missing config blob") - return "", zerr.ErrBadManifest + return zerr.ErrBadManifest } // validate layers - a lightweight check if the blob is present @@ -120,7 +120,7 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy if !ok || err != nil { log.Error().Err(err).Str("digest", layer.Digest.String()).Msg("missing layer blob") - return "", zerr.ErrBadManifest + return zerr.ErrBadManifest } } } @@ -129,49 +129,58 @@ func ValidateManifest(imgStore storageTypes.ImageStore, repo, reference, mediaTy if err := json.Unmarshal(body, &m); err != nil { log.Error().Err(err).Msg("unable to unmarshal JSON") - return "", zerr.ErrBadManifest + return zerr.ErrBadManifest } case ispec.MediaTypeImageIndex: // validate manifest if err := ValidateImageIndexSchema(body); err != nil { log.Error().Err(err).Msg("OCIv1 image index manifest schema validation failed") - return "", zerr.NewError(zerr.ErrBadManifest).AddDetail("jsonSchemaValidation", err.Error()) + return zerr.NewError(zerr.ErrBadManifest).AddDetail("jsonSchemaValidation", err.Error()) } var indexManifest ispec.Index if err := json.Unmarshal(body, &indexManifest); err != nil { log.Error().Err(err).Msg("unable to unmarshal JSON") - return "", zerr.ErrBadManifest + return zerr.ErrBadManifest } for _, manifest := range indexManifest.Manifests { if ok, _, _, err := imgStore.StatBlob(repo, manifest.Digest); !ok || err != nil { log.Error().Err(err).Str("digest", manifest.Digest.String()).Msg("missing manifest blob") - return "", zerr.ErrBadManifest + return zerr.ErrBadManifest } } } - return "", nil + return nil } -func GetAndValidateRequestDigest(body []byte, digestStr string, log zlog.Logger) (godigest.Digest, error) { - bodyDigest := godigest.FromBytes(body) +// Returns the canonical digest or the digest provided by the reference if any +// Per spec, the canonical digest would always be returned to the client in +// request headers, but that does not make sense if the client requested a different digest algorithm +// See https://github.com/opencontainers/distribution-spec/issues/494 +func GetAndValidateRequestDigest(body []byte, reference string, log zlog.Logger) ( + godigest.Digest, error, +) { + expectedDigest, err := godigest.Parse(reference) + if err != nil { + // This is a non-digest reference + return godigest.Canonical.FromBytes(body), err + } + + actualDigest := expectedDigest.Algorithm().FromBytes(body) - d, err := godigest.Parse(digestStr) - if err == nil { - if d.String() != bodyDigest.String() { - log.Error().Str("actual", bodyDigest.String()).Str("expected", d.String()). - Msg("manifest digest is not valid") + if expectedDigest.String() != actualDigest.String() { + log.Error().Str("actual", actualDigest.String()).Str("expected", expectedDigest.String()). + Msg("manifest digest is not valid") - return "", zerr.ErrBadManifest - } + return actualDigest, zerr.ErrBadManifest } - return bodyDigest, err + return actualDigest, nil } /* diff --git a/pkg/storage/gc/gc.go b/pkg/storage/gc/gc.go index c51b9b260d..ebb2e4961b 100644 --- a/pkg/storage/gc/gc.go +++ b/pkg/storage/gc/gc.go @@ -579,11 +579,10 @@ func (gc GarbageCollect) removeUnreferencedBlobs(repo string, delay time.Duratio gcBlobs := make([]godigest.Digest, 0) - for _, blob := range allBlobs { - digest := godigest.NewDigestFromEncoded(godigest.SHA256, blob) + for _, digest := range allBlobs { if err = digest.Validate(); err != nil { - log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", blob). - Msg("unable to parse digest") + log.Error().Err(err).Str("module", "gc").Str("repository", repo). + Str("digest", digest.String()).Msg("unable to parse digest") return err } @@ -591,8 +590,8 @@ func (gc GarbageCollect) removeUnreferencedBlobs(repo string, delay time.Duratio if _, ok := refBlobs[digest.String()]; !ok { canGC, err := isBlobOlderThan(gc.imgStore, repo, digest, delay, log) if err != nil { - log.Error().Err(err).Str("module", "gc").Str("repository", repo).Str("digest", blob). - Msg("unable to determine GC delay") + log.Error().Err(err).Str("module", "gc").Str("repository", repo). + Str("digest", digest.String()).Msg("unable to determine GC delay") return err } diff --git a/pkg/storage/gc/gc_internal_test.go b/pkg/storage/gc/gc_internal_test.go index 5f27ce029e..e88ab2916a 100644 --- a/pkg/storage/gc/gc_internal_test.go +++ b/pkg/storage/gc/gc_internal_test.go @@ -440,8 +440,8 @@ func TestGarbageCollectWithMockedImageStore(t *testing.T) { GetIndexContentFn: func(repo string) ([]byte, error) { return returnedIndexJSONBuf, nil }, - GetAllBlobsFn: func(repo string) ([]string, error) { - return []string{}, errGC + GetAllBlobsFn: func(repo string) ([]godigest.Digest, error) { + return []godigest.Digest{}, errGC }, } diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index a84eb44476..7e5e23bc4f 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -3,7 +3,6 @@ package imagestore import ( "bytes" "context" - "crypto/sha256" "encoding/json" "errors" "fmt" @@ -142,7 +141,7 @@ func (is *ImageStore) initRepo(name string) error { } // create "blobs" subdir - err := is.storeDriver.EnsureDir(path.Join(repoDir, "blobs")) + err := is.storeDriver.EnsureDir(path.Join(repoDir, ispec.ImageBlobsDir)) if err != nil { is.log.Error().Err(err).Msg("error creating blobs subdir") @@ -251,7 +250,7 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) { return false, err } - if filename == "blobs" && !fileInfo.IsDir() { + if filename == ispec.ImageBlobsDir && !fileInfo.IsDir() { return false, nil } @@ -260,7 +259,7 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) { // check blobs dir exists only for filesystem, in s3 we can't have empty dirs if is.storeDriver.Name() == storageConstants.LocalStorageDriverName { - if !is.storeDriver.DirExists(path.Join(dir, "blobs")) { + if !is.storeDriver.DirExists(path.Join(dir, ispec.ImageBlobsDir)) { return false, nil } } @@ -516,9 +515,9 @@ func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //noli refIsDigest = false } - dig, err := common.ValidateManifest(is, repo, reference, mediaType, body, is.log) + err = common.ValidateManifest(is, repo, reference, mediaType, body, is.log) if err != nil { - return dig, "", err + return mDigest, "", err } index, err := common.GetIndex(is, repo, is.log) @@ -573,11 +572,11 @@ func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //noli } if !updateIndex { - return desc.Digest, subjectDigest, nil + return mDigest, subjectDigest, nil } // write manifest to "blobs" - dir := path.Join(is.rootDir, repo, "blobs", mDigest.Algorithm().String()) + dir := path.Join(is.rootDir, repo, ispec.ImageBlobsDir, mDigest.Algorithm().String()) manifestPath := path.Join(dir, mDigest.Encoded()) if _, err = is.storeDriver.WriteFile(manifestPath, body); err != nil { @@ -609,7 +608,7 @@ func (is *ImageStore) PutImageManifest(repo, reference, mediaType string, //noli return "", "", err } - return desc.Digest, subjectDigest, nil + return mDigest, subjectDigest, nil } // DeleteImageManifest deletes the image manifest from the repository. @@ -696,7 +695,8 @@ func (is *ImageStore) deleteImageManifest(repo, reference string, detectCollisio } if toDelete { - p := path.Join(dir, "blobs", manifestDesc.Digest.Algorithm().String(), manifestDesc.Digest.Encoded()) + p := path.Join(dir, ispec.ImageBlobsDir, manifestDesc.Digest.Algorithm().String(), + manifestDesc.Digest.Encoded()) err = is.storeDriver.Delete(p) if err != nil { @@ -882,7 +882,7 @@ func (is *ImageStore) FinishBlobUpload(repo, uuid string, body io.Reader, dstDig return err } - srcDigest, err := getBlobDigest(is, src) + srcDigest, err := getBlobDigest(is, src, dstDigest.Algorithm()) if err != nil { is.log.Error().Err(err).Str("blob", src).Msg("failed to open blob") @@ -896,11 +896,11 @@ func (is *ImageStore) FinishBlobUpload(repo, uuid string, body io.Reader, dstDig return zerr.ErrBadBlobDigest } - dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + dir := path.Join(is.rootDir, repo, ispec.ImageBlobsDir, dstDigest.Algorithm().String()) err = is.storeDriver.EnsureDir(dir) if err != nil { - is.log.Error().Err(err).Msg("error creating blobs/sha256 dir") + is.log.Error().Str("directory", dir).Err(err).Msg("error creating blobs directory") return err } @@ -949,7 +949,16 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, dstDigest godi uuid := u.String() src := is.BlobUploadPath(repo, uuid) - digester := sha256.New() + + dstDigestAlgorithm := dstDigest.Algorithm() + if !dstDigestAlgorithm.Available() { + is.log.Error().Str("dstDigest", dstDigest.String()). + Msg("expected digest algorithm is not supported") + + return "", -1, zerr.ErrBadBlobDigest + } + + digester := dstDigestAlgorithm.Hash() buf := new(bytes.Buffer) _, err = buf.ReadFrom(body) @@ -973,7 +982,7 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, dstDigest godi return "", -1, err } - srcDigest := godigest.NewDigestFromEncoded(godigest.SHA256, fmt.Sprintf("%x", digester.Sum(nil))) + srcDigest := godigest.NewDigestFromEncoded(dstDigestAlgorithm, fmt.Sprintf("%x", digester.Sum(nil))) if srcDigest != dstDigest { is.log.Error().Str("srcDigest", srcDigest.String()). Str("dstDigest", dstDigest.String()).Msg("actual digest not equal to expected digest") @@ -981,7 +990,7 @@ func (is *ImageStore) FullBlobUpload(repo string, body io.Reader, dstDigest godi return "", -1, zerr.ErrBadBlobDigest } - dir := path.Join(is.rootDir, repo, "blobs", dstDigest.Algorithm().String()) + dir := path.Join(is.rootDir, repo, ispec.ImageBlobsDir, dstDigestAlgorithm.String()) _ = is.storeDriver.EnsureDir(dir) var lockLatency time.Time @@ -1129,7 +1138,7 @@ func (is *ImageStore) DeleteBlobUpload(repo, uuid string) error { // BlobPath returns the repository path of a blob. func (is *ImageStore) BlobPath(repo string, digest godigest.Digest) string { - return path.Join(is.rootDir, repo, "blobs", digest.Algorithm().String(), digest.Encoded()) + return path.Join(is.rootDir, repo, ispec.ImageBlobsDir, digest.Algorithm().String(), digest.Encoded()) } /* @@ -1689,7 +1698,8 @@ func (is *ImageStore) deleteBlob(repo string, digest godigest.Digest) error { return nil } -func getBlobDigest(imgStore *ImageStore, path string) (godigest.Digest, error) { +func getBlobDigest(imgStore *ImageStore, path string, digestAlgorithm godigest.Algorithm, +) (godigest.Digest, error) { fileReader, err := imgStore.storeDriver.Reader(path, 0) if err != nil { return "", zerr.ErrUploadNotFound @@ -1697,7 +1707,7 @@ func getBlobDigest(imgStore *ImageStore, path string) (godigest.Digest, error) { defer fileReader.Close() - digest, err := godigest.FromReader(fileReader) + digest, err := digestAlgorithm.FromReader(fileReader) if err != nil { return "", zerr.ErrBadBlobDigest } @@ -1705,24 +1715,37 @@ func getBlobDigest(imgStore *ImageStore, path string) (godigest.Digest, error) { return digest, nil } -func (is *ImageStore) GetAllBlobs(repo string) ([]string, error) { - dir := path.Join(is.rootDir, repo, "blobs", "sha256") +func (is *ImageStore) GetAllBlobs(repo string) ([]godigest.Digest, error) { + blobsDir := path.Join(is.rootDir, repo, ispec.ImageBlobsDir) - files, err := is.storeDriver.List(dir) - if err != nil { - if errors.As(err, &driver.PathNotFoundError{}) { - is.log.Debug().Msg("empty rootDir") + algorithms := []godigest.Algorithm{ + godigest.SHA256, + godigest.SHA384, + godigest.SHA512, + } + + ret := []godigest.Digest{} - return []string{}, nil + for _, algorithm := range algorithms { + dir := path.Join(blobsDir, algorithm.String()) + + files, err := is.storeDriver.List(dir) + if err != nil { + if errors.As(err, &driver.PathNotFoundError{}) { + continue + } + + return []godigest.Digest{}, err } - return []string{}, err + for _, file := range files { + digest := godigest.NewDigestFromEncoded(algorithm, filepath.Base(file)) + ret = append(ret, digest) + } } - ret := []string{} - - for _, file := range files { - ret = append(ret, filepath.Base(file)) + if len(ret) == 0 { + is.log.Debug().Str("directory", blobsDir).Msg("empty blobs directory") } return ret, nil @@ -1751,14 +1774,24 @@ func (is *ImageStore) GetNextDigestWithBlobPaths(repos []string, lastDigests []g if fileInfo.IsDir() { // skip repositories not found in repos repo := path.Base(fileInfo.Path()) - - if !zcommon.Contains(repos, repo) && repo != "blobs" && repo != "sha256" { + if !zcommon.Contains(repos, repo) && + repo != ispec.ImageBlobsDir && + repo != godigest.SHA256.String() && + repo != godigest.SHA384.String() && + repo != godigest.SHA512.String() { return driver.ErrSkipDir } } - blobDigest := godigest.NewDigestFromEncoded("sha256", path.Base(fileInfo.Path())) + digestHash := path.Base(fileInfo.Path()) + digestAlgorithm := godigest.Algorithm(path.Base(path.Dir(fileInfo.Path()))) + + blobDigest := godigest.NewDigestFromEncoded(digestAlgorithm, digestHash) if err := blobDigest.Validate(); err != nil { //nolint: nilerr + is.log.Debug().Str("path", fileInfo.Path()).Str("digestHash", digestHash). + Str("digestAlgorithm", digestAlgorithm.String()). + Msg("digest validation failed when walking blob paths") + return nil //nolint: nilerr // ignore files which are not blobs } diff --git a/pkg/storage/local/local_test.go b/pkg/storage/local/local_test.go index 6f5ec1cc12..87f13989bc 100644 --- a/pkg/storage/local/local_test.go +++ b/pkg/storage/local/local_test.go @@ -2037,7 +2037,8 @@ func TestGarbageCollectForImageStore(t *testing.T) { So(err, ShouldBeNil) manifestDigest := image.ManifestDescriptor.Digest - err = os.Remove(path.Join(dir, repoName, "blobs/sha256", manifestDigest.Encoded())) + err = os.Remove(path.Join(dir, repoName, "blobs", + manifestDigest.Algorithm().String(), manifestDigest.Encoded())) if err != nil { panic(err) } @@ -2232,7 +2233,8 @@ func TestGarbageCollectImageUnknownManifest(t *testing.T) { So(err, ShouldBeNil) artifactDigest := godigest.FromBytes(artifactBuf) - err = os.WriteFile(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", artifactDigest.Encoded()), + err = os.WriteFile(path.Join(imgStore.RootDir(), repoName, "blobs", + artifactDigest.Algorithm().String(), artifactDigest.Encoded()), artifactBuf, storageConstants.DefaultFilePerms) So(err, ShouldBeNil) @@ -2249,7 +2251,8 @@ func TestGarbageCollectImageUnknownManifest(t *testing.T) { So(err, ShouldBeNil) referrerDigest := godigest.FromBytes(referrerBuf) - err = os.WriteFile(path.Join(imgStore.RootDir(), repoName, "blobs", "sha256", referrerDigest.Encoded()), + err = os.WriteFile(path.Join(imgStore.RootDir(), repoName, "blobs", + artifactDigest.Algorithm().String(), referrerDigest.Encoded()), referrerBuf, storageConstants.DefaultFilePerms) So(err, ShouldBeNil) diff --git a/pkg/storage/s3/s3_test.go b/pkg/storage/s3/s3_test.go index 924f5aa80d..0ddea63a1c 100644 --- a/pkg/storage/s3/s3_test.go +++ b/pkg/storage/s3/s3_test.go @@ -2274,7 +2274,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) }, @@ -2290,7 +2290,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Trigger GetContent error in restoreDedupedBlobs()", t, func() { imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) { + if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { return &FileInfoMock{ SizeFn: func() int64 { return int64(0) @@ -2310,7 +2310,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) _ = walkFn(&FileInfoMock{ @@ -2318,7 +2318,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) @@ -2339,7 +2339,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Trigger GetContent error in restoreDedupedBlobs()", t, func() { imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) { + if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { return &FileInfoMock{ SizeFn: func() int64 { return int64(0) @@ -2359,7 +2359,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) _ = walkFn(&FileInfoMock{ @@ -2367,7 +2367,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) @@ -2388,7 +2388,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Trigger Stat() error in restoreDedupedBlobs()", t, func() { imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) { + if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { return &FileInfoMock{ SizeFn: func() int64 { return int64(10) @@ -2408,7 +2408,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) _ = walkFn(&FileInfoMock{ @@ -2416,7 +2416,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) @@ -2433,7 +2433,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Trigger Stat() error in dedupeBlobs()", func() { imgStore := createMockStorage(testDir, t.TempDir(), true, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) { + if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { return &FileInfoMock{ SizeFn: func() int64 { return int64(10) @@ -2453,7 +2453,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) _ = walkFn(&FileInfoMock{ @@ -2461,7 +2461,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) @@ -2481,7 +2481,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { tdir := t.TempDir() imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) { + if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { return &FileInfoMock{ SizeFn: func() int64 { return int64(0) @@ -2501,7 +2501,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) _ = walkFn(&FileInfoMock{ @@ -2509,7 +2509,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) @@ -2532,7 +2532,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { tdir := t.TempDir() imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) { + if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { return &FileInfoMock{ SizeFn: func() int64 { return int64(0) @@ -2552,7 +2552,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) _ = walkFn(&FileInfoMock{ @@ -2560,7 +2560,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) @@ -2600,7 +2600,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) _ = walkFn(&FileInfoMock{ @@ -2608,7 +2608,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) @@ -2638,7 +2638,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { Convey("Trigger cache errors", t, func() { storageDriverMockIfBranch := &StorageDriverMock{ StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) { - if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) { + if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { return &FileInfoMock{ SizeFn: func() int64 { return int64(0) @@ -2658,7 +2658,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) _ = walkFn(&FileInfoMock{ @@ -2666,7 +2666,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) @@ -2696,7 +2696,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) _ = walkFn(&FileInfoMock{ @@ -2704,7 +2704,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PathFn: func() string { - return fmt.Sprintf("path/to/second/%s", validDigest.Encoded()) + return fmt.Sprintf("path/to/second/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) }, }) @@ -2737,7 +2737,7 @@ func TestRebuildDedupeMockStoreDriver(t *testing.T) { return false }, PutBlobFn: func(digest godigest.Digest, path string) error { - if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) { + if path == fmt.Sprintf("path/to/%s/%s", validDigest.Algorithm().String(), validDigest.Encoded()) { return errCache } diff --git a/pkg/storage/scrub.go b/pkg/storage/scrub.go index 19677cd3b6..368acf496b 100644 --- a/pkg/storage/scrub.go +++ b/pkg/storage/scrub.go @@ -286,7 +286,7 @@ func CheckLayers( break } - computedDigest := godigest.FromBytes(layerContent) + computedDigest := layer.Digest.Algorithm().FromBytes(layerContent) if computedDigest != layer.Digest { imageRes = getResult(imageName, tagName, layer.Digest, errors.ErrBadBlobDigest) diff --git a/pkg/storage/types/types.go b/pkg/storage/types/types.go index 692dd7a656..e34b60e90a 100644 --- a/pkg/storage/types/types.go +++ b/pkg/storage/types/types.go @@ -62,7 +62,7 @@ type ImageStore interface { //nolint:interfacebloat RunDedupeBlobs(interval time.Duration, sch *scheduler.Scheduler) RunDedupeForDigest(ctx context.Context, digest godigest.Digest, dedupe bool, duplicateBlobs []string) error GetNextDigestWithBlobPaths(repos []string, lastDigests []godigest.Digest) (godigest.Digest, []string, error) - GetAllBlobs(repo string) ([]string, error) + GetAllBlobs(repo string) ([]godigest.Digest, error) PopulateStorageMetrics(interval time.Duration, sch *scheduler.Scheduler) } diff --git a/pkg/test/image-utils/images.go b/pkg/test/image-utils/images.go index fe53ea1bc8..5031f56284 100644 --- a/pkg/test/image-utils/images.go +++ b/pkg/test/image-utils/images.go @@ -88,9 +88,10 @@ type ManifestBuilder interface { } type Image struct { - Manifest ispec.Manifest - Config ispec.Image - Layers [][]byte + Manifest ispec.Manifest + Config ispec.Image + Layers [][]byte + digestAlgorithm godigest.Algorithm ConfigDescriptor ispec.Descriptor ManifestDescriptor ispec.Descriptor @@ -108,13 +109,28 @@ func (img *Image) Digest() godigest.Digest { panic("unreachable: ispec.Manifest should always be marshable") } - return godigest.FromBytes(blob) + digestAlgorithm := img.digestAlgorithm + + if digestAlgorithm == "" { + digestAlgorithm = godigest.Canonical + } + + return digestAlgorithm.FromBytes(blob) } func (img *Image) DigestStr() string { return img.Digest().String() } +func (img *Image) DigestForAlgorithm(digestAlgorithm godigest.Algorithm) godigest.Digest { + blob, err := json.Marshal(img.Manifest) + if err != nil { + panic("unreachable: ispec.Manifest should always be marshable") + } + + return digestAlgorithm.FromBytes(blob) +} + func (img *Image) Size() int { size := img.ConfigDescriptor.Size + img.ManifestDescriptor.Size @@ -167,7 +183,15 @@ type Layer struct { // specifying the layers of the image. func CreateImageWith() LayerBuilder { // set default values here - return &BaseImageBuilder{} + return &BaseImageBuilder{ + digestAlgorithm: godigest.Canonical, + } +} + +func CreateImageWithDigestAlgorithm(digestAlgorithm godigest.Algorithm) LayerBuilder { + return &BaseImageBuilder{ + digestAlgorithm: digestAlgorithm, + } } func CreateDefaultImage() Image { @@ -223,6 +247,8 @@ type BaseImageBuilder struct { annotations map[string]string subject *ispec.Descriptor artifactType string + + digestAlgorithm godigest.Algorithm } func (ib *BaseImageBuilder) Layers(layers []Layer) ConfigBuilder { @@ -236,7 +262,7 @@ func (ib *BaseImageBuilder) LayerBlobs(layers [][]byte) ConfigBuilder { ib.layers = append(ib.layers, Layer{ Blob: layer, MediaType: ispec.MediaTypeImageLayerGzip, - Digest: godigest.FromBytes(layer), + Digest: ib.digestAlgorithm.FromBytes(layer), }) } @@ -267,7 +293,7 @@ func (ib *BaseImageBuilder) RandomLayers(count, size int) ConfigBuilder { ib.layers = append(ib.layers, Layer{ Blob: layer, MediaType: ispec.MediaTypeImageLayerGzip, - Digest: godigest.FromBytes(layer), + Digest: ib.digestAlgorithm.FromBytes(layer), }) } @@ -290,7 +316,7 @@ func (ib *BaseImageBuilder) VulnerableLayers() VulnerableConfigBuilder { { Blob: layer, MediaType: ispec.MediaTypeImageLayerGzip, - Digest: godigest.FromBytes(layer), + Digest: ib.digestAlgorithm.FromBytes(layer), }, } @@ -309,7 +335,7 @@ func (ib *BaseImageBuilder) ImageConfig(config ispec.Image) ManifestBuilder { MediaType: ispec.MediaTypeImageConfig, Size: int64(len(configBlob)), Data: configBlob, - Digest: godigest.FromBytes(configBlob), + Digest: ib.digestAlgorithm.FromBytes(configBlob), } return ib @@ -351,7 +377,7 @@ func (ib *BaseImageBuilder) CustomConfigBlob(configBlob []byte, mediaType string MediaType: mediaType, Size: int64(len(configBlob)), Data: configBlob, - Digest: godigest.FromBytes(configBlob), + Digest: ib.digestAlgorithm.FromBytes(configBlob), } return ib @@ -372,7 +398,7 @@ func (ib *BaseImageBuilder) RandomConfig() ManifestBuilder { ib.configDescriptor = ispec.Descriptor{ MediaType: ispec.MediaTypeImageConfig, - Digest: godigest.FromBytes(configBlob), + Digest: ib.digestAlgorithm.FromBytes(configBlob), Size: int64(len(configBlob)), Data: configBlob, } @@ -390,7 +416,7 @@ func (ib *BaseImageBuilder) DefaultVulnConfig() ManifestBuilder { vulnConfigDescriptor := ispec.Descriptor{ MediaType: ispec.MediaTypeImageConfig, - Digest: godigest.FromBytes(configBlob), + Digest: ib.digestAlgorithm.FromBytes(configBlob), Size: int64(len(configBlob)), Data: configBlob, } @@ -421,7 +447,7 @@ func (ib *BaseImageBuilder) VulnerableConfig(config ispec.Image) ManifestBuilder vulnConfigDescriptor := ispec.Descriptor{ MediaType: ispec.MediaTypeImageConfig, - Digest: godigest.FromBytes(configBlob), + Digest: ib.digestAlgorithm.FromBytes(configBlob), Size: int64(len(configBlob)), Data: configBlob, } @@ -446,7 +472,7 @@ func (ib *BaseImageBuilder) RandomVulnConfig() ManifestBuilder { vulnConfigDescriptor := ispec.Descriptor{ MediaType: ispec.MediaTypeImageConfig, - Digest: godigest.FromBytes(configBlob), + Digest: ib.digestAlgorithm.FromBytes(configBlob), Size: int64(len(configBlob)), Data: configBlob, } @@ -493,6 +519,7 @@ func (ib *BaseImageBuilder) Build() Image { Subject: ib.subject, Annotations: ib.annotations, }, + digestAlgorithm: ib.digestAlgorithm, } manifestBlob, err := json.Marshal(img.Manifest) @@ -502,7 +529,7 @@ func (ib *BaseImageBuilder) Build() Image { img.ManifestDescriptor = ispec.Descriptor{ MediaType: ispec.MediaTypeImageManifest, - Digest: godigest.FromBytes(manifestBlob), + Digest: ib.digestAlgorithm.FromBytes(manifestBlob), Size: int64(len(manifestBlob)), Data: manifestBlob, } diff --git a/pkg/test/image-utils/multiarch.go b/pkg/test/image-utils/multiarch.go index ffc28f00e3..0b2af7d0b6 100644 --- a/pkg/test/image-utils/multiarch.go +++ b/pkg/test/image-utils/multiarch.go @@ -11,8 +11,9 @@ import ( ) type MultiarchImage struct { - Index ispec.Index - Images []Image + Index ispec.Index + Images []Image + digestAlgorithm godigest.Algorithm IndexDescriptor ispec.Descriptor } @@ -23,13 +24,28 @@ func (mi *MultiarchImage) Digest() godigest.Digest { panic("unreachable: ispec.Index should always be marshable") } - return godigest.FromBytes(indexBlob) + digestAlgorithm := mi.digestAlgorithm + + if digestAlgorithm == "" { + digestAlgorithm = godigest.Canonical + } + + return digestAlgorithm.FromBytes(indexBlob) } func (mi *MultiarchImage) DigestStr() string { return mi.Digest().String() } +func (mi *MultiarchImage) DigestForAlgorithm(digestAlgorithm godigest.Algorithm) godigest.Digest { + blob, err := json.Marshal(mi.Index) + if err != nil { + panic("unreachable: ispec.Index should always be marshable") + } + + return digestAlgorithm.FromBytes(blob) +} + func (mi MultiarchImage) AsImageMeta() mTypes.ImageMeta { index := mi.Index @@ -61,7 +77,15 @@ type MultiarchBuilder interface { } func CreateMultiarchWith() ImagesBuilder { - return &BaseMultiarchBuilder{} + return &BaseMultiarchBuilder{ + digestAlgorithm: godigest.Canonical, + } +} + +func CreateMultiarchWithDigestAlgorithm(digestAlgorithm godigest.Algorithm) ImagesBuilder { + return &BaseMultiarchBuilder{ + digestAlgorithm: digestAlgorithm, + } } func CreateRandomMultiarch() MultiarchImage { @@ -85,10 +109,11 @@ func CreateVulnerableMultiarch() MultiarchImage { } type BaseMultiarchBuilder struct { - images []Image - subject *ispec.Descriptor - artifactType string - annotations map[string]string + images []Image + subject *ispec.Descriptor + artifactType string + annotations map[string]string + digestAlgorithm godigest.Algorithm } func (mb *BaseMultiarchBuilder) Images(images []Image) MultiarchBuilder { @@ -154,7 +179,7 @@ func (mb *BaseMultiarchBuilder) Build() MultiarchImage { panic("unreachable: ispec.Index should always be marshable") } - indexDigest := godigest.FromBytes(indexBlob) + indexDigest := mb.digestAlgorithm.FromBytes(indexBlob) return MultiarchImage{ Index: index, diff --git a/pkg/test/image-utils/upload.go b/pkg/test/image-utils/upload.go index bda283f046..b23baa0e3a 100644 --- a/pkg/test/image-utils/upload.go +++ b/pkg/test/image-utils/upload.go @@ -21,6 +21,12 @@ var ( ) func UploadImage(img Image, baseURL, repo, ref string) error { + digestAlgorithm := img.digestAlgorithm + + if digestAlgorithm == "" { + digestAlgorithm = godigest.Canonical + } + for _, blob := range img.Layers { resp, err := resty.R().Post(baseURL + "/v2/" + repo + "/blobs/uploads/") if err != nil { @@ -33,7 +39,7 @@ func UploadImage(img Image, baseURL, repo, ref string) error { loc := resp.Header().Get("Location") - digest := godigest.FromBytes(blob).String() + digest := digestAlgorithm.FromBytes(blob).String() resp, err = resty.R(). SetHeader("Content-Length", fmt.Sprintf("%d", len(blob))). @@ -63,7 +69,7 @@ func UploadImage(img Image, baseURL, repo, ref string) error { } } - cdigest := godigest.FromBytes(cblob) + cdigest := digestAlgorithm.FromBytes(cblob) if img.Manifest.Config.MediaType == ispec.MediaTypeEmptyJSON || img.Manifest.Config.Digest == ispec.DescriptorEmptyJSON.Digest { @@ -117,14 +123,16 @@ func UploadImage(img Image, baseURL, repo, ref string) error { return ErrPutBlob } - if inject.ErrStatusCode(resp.StatusCode()) != http.StatusCreated { - return ErrPutBlob - } - return err } func UploadImageWithBasicAuth(img Image, baseURL, repo, ref, user, password string) error { + digestAlgorithm := img.digestAlgorithm + + if digestAlgorithm == "" { + digestAlgorithm = godigest.Canonical + } + for _, blob := range img.Layers { resp, err := resty.R(). SetBasicAuth(user, password). @@ -139,7 +147,7 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, ref, user, password stri loc := resp.Header().Get("Location") - digest := godigest.FromBytes(blob).String() + digest := digestAlgorithm.FromBytes(blob).String() resp, err = resty.R(). SetBasicAuth(user, password). @@ -163,7 +171,7 @@ func UploadImageWithBasicAuth(img Image, baseURL, repo, ref, user, password stri return err } - cdigest := godigest.FromBytes(cblob) + cdigest := digestAlgorithm.FromBytes(cblob) if img.Manifest.Config.MediaType == ispec.MediaTypeEmptyJSON { cblob = ispec.DescriptorEmptyJSON.Data diff --git a/pkg/test/image-utils/write.go b/pkg/test/image-utils/write.go index 5bb089c634..90498a982f 100644 --- a/pkg/test/image-utils/write.go +++ b/pkg/test/image-utils/write.go @@ -18,9 +18,15 @@ func WriteImageToFileSystem(image Image, repoName, ref string, storeController s return err } + digestAlgorithm := image.digestAlgorithm + + if digestAlgorithm == "" { + digestAlgorithm = godigest.Canonical + } + for _, layerBlob := range image.Layers { layerReader := bytes.NewReader(layerBlob) - layerDigest := godigest.FromBytes(layerBlob) + layerDigest := digestAlgorithm.FromBytes(layerBlob) _, _, err = store.FullBlobUpload(repoName, layerReader, layerDigest) if err != nil { @@ -34,7 +40,7 @@ func WriteImageToFileSystem(image Image, repoName, ref string, storeController s } configReader := bytes.NewReader(configBlob) - configDigest := godigest.FromBytes(configBlob) + configDigest := digestAlgorithm.FromBytes(configBlob) _, _, err = store.FullBlobUpload(repoName, configReader, configDigest) if err != nil { diff --git a/pkg/test/mocks/image_store_mock.go b/pkg/test/mocks/image_store_mock.go index 0c65316a73..33cc763a6a 100644 --- a/pkg/test/mocks/image_store_mock.go +++ b/pkg/test/mocks/image_store_mock.go @@ -54,7 +54,7 @@ type MockedImageStore struct { RunDedupeForDigestFn func(ctx context.Context, digest godigest.Digest, dedupe bool, duplicateBlobs []string) error GetNextDigestWithBlobPathsFn func(repos []string, lastDigests []godigest.Digest) (godigest.Digest, []string, error) - GetAllBlobsFn func(repo string) ([]string, error) + GetAllBlobsFn func(repo string) ([]godigest.Digest, error) CleanupRepoFn func(repo string, blobs []godigest.Digest, removeRepo bool) (int, error) PutIndexContentFn func(repo string, index ispec.Index) error PopulateStorageMetricsFn func(interval time.Duration, sch *scheduler.Scheduler) @@ -166,12 +166,12 @@ func (is MockedImageStore) GetImageTags(name string) ([]string, error) { return []string{}, nil } -func (is MockedImageStore) GetAllBlobs(repo string) ([]string, error) { +func (is MockedImageStore) GetAllBlobs(repo string) ([]godigest.Digest, error) { if is.GetAllBlobsFn != nil { return is.GetAllBlobsFn(repo) } - return []string{}, nil + return []godigest.Digest{}, nil } func (is MockedImageStore) DeleteImageManifest(name string, reference string, detectCollision bool) error {