From 5ae5b9255b1393d0cdbad9b4aff9281092ea3ef5 Mon Sep 17 00:00:00 2001 From: Eusebiu Petu Date: Mon, 4 Nov 2024 20:22:32 +0200 Subject: [PATCH] fix(sync): use pagination when querying remote catalog feat(api): added /v2/_catalog pagination, fixes #2715 Signed-off-by: Eusebiu Petu --- pkg/api/authz.go | 15 + pkg/api/controller_test.go | 434 ++++++++++++++++++ pkg/api/routes.go | 128 ++++-- pkg/api/routes_test.go | 30 +- pkg/extensions/sync/httpclient/client.go | 15 +- pkg/extensions/sync/references/cosign.go | 2 +- pkg/extensions/sync/references/oci.go | 4 +- pkg/extensions/sync/references/references.go | 2 +- .../sync/references/referrers_tag.go | 2 +- pkg/extensions/sync/remote.go | 29 +- pkg/storage/imagestore/imagestore.go | 83 ++++ pkg/storage/storage_controller.go | 20 +- pkg/storage/storage_test.go | 23 + pkg/storage/types/types.go | 3 + pkg/test/mocks/image_store_mock.go | 32 +- test/blackbox/pushpull.bats | 13 + 16 files changed, 768 insertions(+), 67 deletions(-) diff --git a/pkg/api/authz.go b/pkg/api/authz.go index efb91c63f..c41e0edc1 100644 --- a/pkg/api/authz.go +++ b/pkg/api/authz.go @@ -12,6 +12,7 @@ import ( "zotregistry.dev/zot/pkg/common" "zotregistry.dev/zot/pkg/log" reqCtx "zotregistry.dev/zot/pkg/requestcontext" + storageTypes "zotregistry.dev/zot/pkg/storage/types" ) const ( @@ -20,6 +21,20 @@ const ( OPENID = "OpenID" ) +func AuthzFilterFunc(userAc *reqCtx.UserAccessControl) storageTypes.FilterRepoFunc { + return func(repo string) (bool, error) { + if userAc == nil { + return true, nil + } + + if userAc.Can(constants.ReadPermission, repo) { + return true, nil + } + + return false, nil + } +} + // AccessController authorizes users to act on resources. type AccessController struct { Config *config.AccessControlConfig diff --git a/pkg/api/controller_test.go b/pkg/api/controller_test.go index d97d6b4cf..9b8d4503a 100644 --- a/pkg/api/controller_test.go +++ b/pkg/api/controller_test.go @@ -8145,6 +8145,440 @@ func TestRouteFailures(t *testing.T) { }) } +func TestPagedRepositoriesWithAuthorization(t *testing.T) { + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + username, _ := test.GenerateRandomString() + password, _ := test.GenerateRandomString() + htpasswdPath := test.MakeHtpasswdFileFromString(test.GetCredString(username, password)) + + defer os.Remove(htpasswdPath) + + conf.HTTP.Auth = &config.AuthConfig{ + HTPasswd: config.AuthHTPasswd{ + Path: htpasswdPath, + }, + } + + readPolicyGroup := config.PolicyGroup{ + Policies: []config.Policy{ + { + Users: []string{username}, + Actions: []string{ + constants.ReadPermission, + }, + }, + }, + DefaultPolicy: []string{}, + } + + conf.HTTP.AccessControl = &config.AccessControlConfig{ + Repositories: config.Repositories{ + test.AuthorizationAllRepos: config.PolicyGroup{ + Policies: []config.Policy{ + { + Users: []string{username}, + Actions: []string{ + constants.ReadPermission, + constants.CreatePermission, + }, + }, + }, + + DefaultPolicy: []string{}, + }, + }, + AdminPolicy: config.Policy{ + Users: []string{}, + Actions: []string{}, + }, + } + + ctlr := api.NewController(conf) + ctlr.Config.Storage.RootDirectory = t.TempDir() + + cm := test.NewControllerManager(ctlr) + cm.StartAndWait(port) + + defer cm.StopServer() + + client := resty.New() + client.SetBasicAuth(username, password) + + img := CreateRandomImage() + + tag := "0.1" + repoNames := []string{ + "alpine1", "alpine2", "alpine3", + "alpine4", "alpine5", "alpine6", + "alpine7", "alpine8", "alpine9", + } + + for _, repo := range repoNames { + err := UploadImageWithBasicAuth(img, baseURL, repo, tag, username, password) + if err != nil { + panic(err) + } + } + + conf.HTTP.AccessControl.Repositories = config.Repositories{ + "alpine[13579]": readPolicyGroup, + "alpine[2468]": config.PolicyGroup{}, + } + + // Note empty strings signify the query parameter is not set + // There are separate tests for passing the empty string as query parameter + testCases := []struct { + testCaseName string + pageSize string + last string + expectedRepos []string + }{ + { + testCaseName: "no parameters", + pageSize: "", + last: "", + expectedRepos: []string{"alpine1", "alpine3", "alpine5", "alpine7", "alpine9"}, + }, + { + testCaseName: "first 3", + pageSize: "3", + last: "", + expectedRepos: []string{"alpine1", "alpine3", "alpine5"}, + }, + { + testCaseName: "next 2", + pageSize: "3", + last: "alpine5", + expectedRepos: []string{"alpine7", "alpine9"}, + }, + { + testCaseName: "0", + pageSize: "0", + last: "", + expectedRepos: []string{}, + }, + { + testCaseName: "Test the parameter 'last' without parameter 'n'", + pageSize: "", + last: "alpine3", + expectedRepos: []string{"alpine5", "alpine7", "alpine9"}, + }, + { + testCaseName: "Test the parameter 'last' with the final repo as value", + pageSize: "", + last: "alpine9", + expectedRepos: []string{}, + }, + } + + for _, testCase := range testCases { + Convey(testCase.testCaseName, t, func() { + testHTTPPagedRepositories(t, client, baseURL, testCase.testCaseName, testCase.pageSize, + testCase.last, testCase.expectedRepos, repoNames[len(repoNames)-1]) + }) + } +} + +func TestPagedRepositoriesWithSubpaths(t *testing.T) { + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + + dir := t.TempDir() + firstSubDir := t.TempDir() + secondSubDir := t.TempDir() + + subPaths := make(map[string]config.StorageConfig) + + subPaths["/a"] = config.StorageConfig{RootDirectory: firstSubDir} + subPaths["/b"] = config.StorageConfig{RootDirectory: secondSubDir} + + ctlr := makeController(conf, dir) + ctlr.Config.Storage.SubPaths = subPaths + ctlr.Config.Storage.Commit = true + + cm := test.NewControllerManager(ctlr) + cm.StartAndWait(port) + + defer cm.StopServer() + + rthdlr := api.NewRouteHandler(ctlr) + + img := CreateRandomImage() + + tag := "0.1" + repoNames := []string{ + "alpine1", "alpine2", "alpine3", + "a/alpine4", "a/alpine5", "a/alpine6", + "b/alpine7", "b/alpine8", "b/alpine9", + } + + for _, repo := range repoNames { + err := UploadImage(img, baseURL, repo, tag) + if err != nil { + panic(err) + } + } + + // Note empty strings signify the query parameter is not set + // There are separate tests for passing the empty string as query parameter + testCases := []struct { + testCaseName string + pageSize string + last string + expectedRepos []string + }{ + { + testCaseName: "no parameters", + pageSize: "", + last: "", + expectedRepos: repoNames, + }, + { + testCaseName: "first 5", + pageSize: "5", + last: "", + expectedRepos: repoNames[:5], + }, + { + testCaseName: "next 5", + pageSize: "5", + last: "a/alpine5", + expectedRepos: repoNames[5:9], + }, + { + testCaseName: "0", + pageSize: "0", + last: "", + expectedRepos: []string{}, + }, + { + testCaseName: "Test the parameter 'last' without parameter 'n'", + pageSize: "", + last: "alpine2", + expectedRepos: repoNames[2:9], + }, + { + testCaseName: "Test the parameter 'last' with the final repo as value", + pageSize: "", + last: repoNames[len(repoNames)-1], + expectedRepos: []string{}, + }, + } + + for _, testCase := range testCases { + Convey(testCase.testCaseName, t, func() { + testPagedRepositories(t, rthdlr, baseURL, testCase.testCaseName, testCase.pageSize, + testCase.last, testCase.expectedRepos, repoNames[len(repoNames)-1]) + }) + } +} + +func TestPagedRepositories(t *testing.T) { + port := test.GetFreePort() + baseURL := test.GetBaseURL(port) + conf := config.New() + conf.HTTP.Port = port + + ctlr := makeController(conf, t.TempDir()) + ctlr.Config.Storage.Commit = true + + cm := test.NewControllerManager(ctlr) + cm.StartAndWait(port) + + defer cm.StopServer() + + rthdlr := api.NewRouteHandler(ctlr) + + img := CreateRandomImage() + + tag := "0.1" + repoName := "alpine" + repoNames := []string{ + "alpine1", "alpine2", "alpine3", + "alpine4", "alpine5", "alpine6", + "alpine7", "alpine8", "alpine9", + } + + for _, repo := range repoNames { + err := UploadImage(img, baseURL, repo, tag) + if err != nil { + panic(err) + } + } + + // Note empty strings signify the query parameter is not set + // There are separate tests for passing the empty string as query parameter + testCases := []struct { + testCaseName string + pageSize string + last string + expectedRepos []string + }{ + { + testCaseName: "no parameters", + pageSize: "", + last: "", + expectedRepos: repoNames, + }, + { + testCaseName: "first 5", + pageSize: "5", + last: "", + expectedRepos: repoNames[:5], + }, + { + testCaseName: "next 5", + pageSize: "5", + last: repoName + "5", + expectedRepos: repoNames[5:9], + }, + { + testCaseName: "0", + pageSize: "0", + last: "", + expectedRepos: []string{}, + }, + { + testCaseName: "Test the parameter 'last' without parameter 'n'", + pageSize: "", + last: repoName + "2", + expectedRepos: repoNames[2:9], + }, + { + testCaseName: "Test the parameter 'last' with the final repo as value", + pageSize: "", + last: repoName + "9", + expectedRepos: []string{}, + }, + } + + for _, testCase := range testCases { + Convey(testCase.testCaseName, t, func() { + testPagedRepositories(t, rthdlr, baseURL, testCase.testCaseName, testCase.pageSize, + testCase.last, testCase.expectedRepos, repoNames[len(repoNames)-1]) + }) + } +} + +func testHTTPPagedRepositories(t *testing.T, client *resty.Client, baseURL string, testCaseName string, + pageSize string, + last string, + expectedRepos []string, lastRepoInStorage string, +) { + Convey(testCaseName, func() { + t.Log("Running " + testCaseName) + + params := make(map[string]string) + + if pageSize != "" || last != "" { + if pageSize != "" { + params["n"] = pageSize + } + + if last != "" { + params["last"] = last + } + } + + resp, err := client.R().SetQueryParams(params).Get(baseURL + "/v2/_catalog") + So(err, ShouldBeNil) + So(resp, ShouldNotBeNil) + So(resp.StatusCode(), ShouldEqual, http.StatusOK) + + catalog := struct { + Repositories []string `json:"repositories"` + }{} + + err = json.Unmarshal(resp.Body(), &catalog) + So(err, ShouldBeNil) + + So(catalog.Repositories, ShouldEqual, expectedRepos) + + actualLinkValue := resp.Header().Get("Link") + if pageSize == "0" || pageSize == "" { //nolint:gocritic + So(actualLinkValue, ShouldEqual, "") + } else if expectedRepos[len(expectedRepos)-1] == catalog.Repositories[len(catalog.Repositories)-1] && + expectedRepos[len(expectedRepos)-1] == lastRepoInStorage { + So(actualLinkValue, ShouldEqual, "") + } else { + expectedLinkValue := fmt.Sprintf("; rel=\"next\"", + pageSize, catalog.Repositories[len(catalog.Repositories)-1], + ) + So(actualLinkValue, ShouldEqual, expectedLinkValue) + } + + t.Log("Finished " + testCaseName) + }) +} + +func testPagedRepositories(t *testing.T, rthdlr *api.RouteHandler, baseURL string, testCaseName string, + pageSize string, + last string, + expectedRepos []string, lastRepoInStorage string, +) { + Convey(testCaseName, func() { + t.Log("Running " + testCaseName) + + request, _ := http.NewRequestWithContext(context.TODO(), http.MethodGet, + baseURL+constants.RoutePrefix+constants.ExtCatalogPrefix, nil) + + if pageSize != "" || last != "" { + qparm := request.URL.Query() + + if pageSize != "" { + qparm.Add("n", pageSize) + } + + if last != "" { + qparm.Add("last", last) + } + + request.URL.RawQuery = qparm.Encode() + } + + response := httptest.NewRecorder() + + rthdlr.ListRepositories(response, request) + + resp := response.Result() + defer resp.Body.Close() + So(resp, ShouldNotBeNil) + So(resp.StatusCode, ShouldEqual, http.StatusOK) + + catalog := struct { + Repositories []string `json:"repositories"` + }{} + + body, err := io.ReadAll(resp.Body) + So(err, ShouldBeNil) + + err = json.Unmarshal(body, &catalog) + So(err, ShouldBeNil) + + So(catalog.Repositories, ShouldEqual, expectedRepos) + + actualLinkValue := resp.Header.Get("Link") + if pageSize == "0" || pageSize == "" { //nolint:gocritic + So(actualLinkValue, ShouldEqual, "") + } else if expectedRepos[len(expectedRepos)-1] == catalog.Repositories[len(catalog.Repositories)-1] && + expectedRepos[len(expectedRepos)-1] == lastRepoInStorage { + So(actualLinkValue, ShouldEqual, "") + } else { + expectedLinkValue := fmt.Sprintf("; rel=\"next\"", + pageSize, catalog.Repositories[len(catalog.Repositories)-1], + ) + So(actualLinkValue, ShouldEqual, expectedLinkValue) + } + + t.Log("Finished " + testCaseName) + }) +} + func TestListingTags(t *testing.T) { port := test.GetFreePort() baseURL := test.GetBaseURL(port) diff --git a/pkg/api/routes.go b/pkg/api/routes.go index 164b6bf9a..b92c42d56 100644 --- a/pkg/api/routes.go +++ b/pkg/api/routes.go @@ -47,6 +47,7 @@ import ( mTypes "zotregistry.dev/zot/pkg/meta/types" zreg "zotregistry.dev/zot/pkg/regexp" reqCtx "zotregistry.dev/zot/pkg/requestcontext" + "zotregistry.dev/zot/pkg/storage" storageCommon "zotregistry.dev/zot/pkg/storage/common" storageTypes "zotregistry.dev/zot/pkg/storage/types" "zotregistry.dev/zot/pkg/test/inject" @@ -1763,6 +1764,81 @@ type RepositoryList struct { Repositories []string `json:"repositories"` } +func (rh *RouteHandler) listStorageRepositories(lastEntry string, maxEntries int, + userAc *reqCtx.UserAccessControl, +) ([]string, bool, error) { + var moreEntries bool + + var err error + + var repos []string + + remainder := maxEntries + + combineRepoList := make([]string, 0) + + subStore := rh.c.StoreController.SubStore + + subPaths := make([]string, 0) + for subPath := range subStore { + subPaths = append(subPaths, subPath) + } + + sort.Strings(subPaths) + + storePath := rh.c.StoreController.GetStorePath(lastEntry) + if storePath == storage.DefaultStorePath { + singleStore := rh.c.StoreController.DefaultStore + + repos, moreEntries, err = singleStore.GetNextRepositories(lastEntry, remainder, AuthzFilterFunc(userAc)) + if err != nil { + return repos, false, err + } + + remainder = maxEntries - len(repos) + + if moreEntries && remainder <= 0 && len(repos) > 0 { + // maxEntries has been hit + lastEntry = repos[len(repos)-1] + } else { + // reset for the next substores + lastEntry = "" + } + + combineRepoList = append(combineRepoList, repos...) + } + + for _, subPath := range subPaths { + imgStore := subStore[subPath] + + if lastEntry != "" && subPath != storePath { + continue + } + + if remainder > 0 || maxEntries == -1 { + repos, moreEntries, err = imgStore.GetNextRepositories(lastEntry, remainder, AuthzFilterFunc(userAc)) + if err != nil { + return combineRepoList, false, err + } + + // compute remainder + remainder -= len(repos) + + if moreEntries && remainder <= 0 && len(repos) > 0 { + // maxEntries has been hit + lastEntry = repos[len(repos)-1] + } else { + // reset for the next substores + lastEntry = "" + } + + combineRepoList = append(combineRepoList, repos...) + } + } + + return combineRepoList, moreEntries, nil +} + // ListRepositories godoc // @Summary List image repositories // @Description List all image repositories @@ -1776,50 +1852,40 @@ func (rh *RouteHandler) ListRepositories(response http.ResponseWriter, request * return } - combineRepoList := make([]string, 0) - - subStore := rh.c.StoreController.SubStore - - for _, imgStore := range subStore { - repos, err := imgStore.GetRepositories() - if err != nil { - response.WriteHeader(http.StatusInternalServerError) + q := request.URL.Query() - return - } + lastEntry := q.Get("last") - combineRepoList = append(combineRepoList, repos...) + maxEntries, err := strconv.Atoi(q.Get("n")) + if err != nil { + maxEntries = -1 } - singleStore := rh.c.StoreController.DefaultStore - if singleStore != nil { - repos, err := singleStore.GetRepositories() - if err != nil { - response.WriteHeader(http.StatusInternalServerError) - - return - } + // authz context + userAc, err := reqCtx.UserAcFromContext(request.Context()) + if err != nil { + response.WriteHeader(http.StatusInternalServerError) - combineRepoList = append(combineRepoList, repos...) + return } - repos := make([]string, 0) - // authz context - userAc, err := reqCtx.UserAcFromContext(request.Context()) + repos, moreEntries, err := rh.listStorageRepositories(lastEntry, maxEntries, userAc) if err != nil { response.WriteHeader(http.StatusInternalServerError) return } - if userAc != nil { - for _, r := range combineRepoList { - if userAc.Can(constants.ReadPermission, r) { - repos = append(repos, r) - } - } - } else { - repos = combineRepoList + if moreEntries && len(repos) > 0 { + lastRepo := repos[len(repos)-1] + + response.Header().Set( + "Link", + fmt.Sprintf("; rel=\"next\"", + maxEntries, + lastRepo, + ), + ) } is := RepositoryList{Repositories: repos} diff --git a/pkg/api/routes_test.go b/pkg/api/routes_test.go index bc1554db5..68f9c3ef7 100644 --- a/pkg/api/routes_test.go +++ b/pkg/api/routes_test.go @@ -1359,8 +1359,10 @@ func TestRoutes(t *testing.T) { "session_id": "test", }, &mocks.MockedImageStore{ - GetRepositoriesFn: func() ([]string, error) { - return []string{}, ErrUnexpectedError + GetNextRepositoriesFn: func(lastRepo string, maxEntries int, + fn storageTypes.FilterRepoFunc, + ) ([]string, bool, error) { + return []string{}, false, ErrUnexpectedError }, }, ) @@ -1374,8 +1376,10 @@ func TestRoutes(t *testing.T) { "session_id": "test", }, &mocks.MockedImageStore{ - GetRepositoriesFn: func() ([]string, error) { - return []string{}, ErrUnexpectedError + GetNextRepositoriesFn: func(lastRepo string, maxEntries int, + fn storageTypes.FilterRepoFunc, + ) ([]string, bool, error) { + return []string{}, false, ErrUnexpectedError }, }, ) @@ -1384,19 +1388,25 @@ func TestRoutes(t *testing.T) { Convey("ListRepositories with Authz", func() { ctlr.StoreController.DefaultStore = &mocks.MockedImageStore{ - GetRepositoriesFn: func() ([]string, error) { - return []string{"repo"}, nil + GetNextRepositoriesFn: func(lastRepo string, maxEntries int, + fn storageTypes.FilterRepoFunc, + ) ([]string, bool, error) { + return []string{"repo"}, false, nil }, } ctlr.StoreController.SubStore = map[string]storageTypes.ImageStore{ "test1": &mocks.MockedImageStore{ - GetRepositoriesFn: func() ([]string, error) { - return []string{"repo1"}, nil + GetNextRepositoriesFn: func(lastRepo string, maxEntries int, + fn storageTypes.FilterRepoFunc, + ) ([]string, bool, error) { + return []string{"repo1"}, false, nil }, }, "test2": &mocks.MockedImageStore{ - GetRepositoriesFn: func() ([]string, error) { - return []string{"repo2"}, nil + GetNextRepositoriesFn: func(lastRepo string, maxEntries int, + fn storageTypes.FilterRepoFunc, + ) ([]string, bool, error) { + return []string{"repo2"}, false, nil }, }, } diff --git a/pkg/extensions/sync/httpclient/client.go b/pkg/extensions/sync/httpclient/client.go index 37968cab7..4517a3621 100644 --- a/pkg/extensions/sync/httpclient/client.go +++ b/pkg/extensions/sync/httpclient/client.go @@ -177,9 +177,9 @@ func (httpClient *Client) Ping() bool { return false } -func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interface{}, mediaType string, +func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interface{}, mediaType string, rawQuery string, route ...string, -) ([]byte, string, int, error) { +) ([]byte, http.Header, int, error) { httpClient.lock.RLock() defer httpClient.lock.RUnlock() @@ -192,11 +192,12 @@ func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interfac // we know that the second route argument is always the repo name. // need it for caching tokens, it's not used in requests made to authz server. if idx == 1 { - namespace = path + namespace = strings.Trim(path, "/") } } - url.RawQuery = url.Query().Encode() + url.RawQuery = rawQuery + //nolint: bodyclose,contextcheck resp, body, err := httpClient.makeAndDoRequest(http.MethodGet, mediaType, namespace, url.String()) if err != nil { @@ -204,11 +205,11 @@ func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interfac Str("errorType", common.TypeOf(err)). Msg("failed to make request") - return nil, "", -1, err + return nil, nil, -1, err } if resp.StatusCode != http.StatusOK { - return nil, "", resp.StatusCode, errors.New(string(body)) //nolint:goerr113 + return nil, nil, resp.StatusCode, errors.New(string(body)) //nolint:goerr113 } // read blob @@ -216,7 +217,7 @@ func (httpClient *Client) MakeGetRequest(ctx context.Context, resultPtr interfac err = json.Unmarshal(body, &resultPtr) } - return body, resp.Header.Get("Content-Type"), resp.StatusCode, err + return body, resp.Header, resp.StatusCode, err } func (httpClient *Client) getAuthType(resp *http.Response) { diff --git a/pkg/extensions/sync/references/cosign.go b/pkg/extensions/sync/references/cosign.go index eaa6008cf..160b8247c 100644 --- a/pkg/extensions/sync/references/cosign.go +++ b/pkg/extensions/sync/references/cosign.go @@ -172,7 +172,7 @@ func (ref CosignReference) SyncReferences(ctx context.Context, localRepo, remote func (ref CosignReference) getManifest(ctx context.Context, repo, cosignTag string) (*ispec.Manifest, []byte, error) { var cosignManifest ispec.Manifest - body, _, statusCode, err := ref.client.MakeGetRequest(ctx, &cosignManifest, ispec.MediaTypeImageManifest, + body, _, statusCode, err := ref.client.MakeGetRequest(ctx, &cosignManifest, ispec.MediaTypeImageManifest, "", "v2", repo, "manifests", cosignTag) if err != nil { if statusCode == http.StatusNotFound { diff --git a/pkg/extensions/sync/references/oci.go b/pkg/extensions/sync/references/oci.go index 16c78f4d3..fc391abd7 100644 --- a/pkg/extensions/sync/references/oci.go +++ b/pkg/extensions/sync/references/oci.go @@ -159,7 +159,7 @@ func (ref OciReferences) SyncReferences(ctx context.Context, localRepo, remoteRe func (ref OciReferences) getIndex(ctx context.Context, repo, subjectDigestStr string) (ispec.Index, error) { var index ispec.Index - _, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex, + _, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex, "", "v2", repo, "referrers", subjectDigestStr) if err != nil { if statusCode == http.StatusNotFound { @@ -182,7 +182,7 @@ func syncManifest(ctx context.Context, client *client.Client, imageStore storage var refDigest godigest.Digest - OCIRefBuf, _, statusCode, err := client.MakeGetRequest(ctx, &manifest, ispec.MediaTypeImageManifest, + OCIRefBuf, _, statusCode, err := client.MakeGetRequest(ctx, &manifest, ispec.MediaTypeImageManifest, "", "v2", remoteRepo, "manifests", desc.Digest.String()) if err != nil { if statusCode == http.StatusNotFound { diff --git a/pkg/extensions/sync/references/references.go b/pkg/extensions/sync/references/references.go index 3e7ec2748..17efc21af 100644 --- a/pkg/extensions/sync/references/references.go +++ b/pkg/extensions/sync/references/references.go @@ -152,7 +152,7 @@ func syncBlob(ctx context.Context, client *client.Client, imageStore storageType ) error { var resultPtr interface{} - body, _, statusCode, err := client.MakeGetRequest(ctx, resultPtr, "", "v2", remoteRepo, "blobs", digest.String()) + body, _, statusCode, err := client.MakeGetRequest(ctx, resultPtr, "", "", "v2", remoteRepo, "blobs", digest.String()) if err != nil { if statusCode != http.StatusOK { log.Info().Str("repo", remoteRepo).Str("digest", digest.String()).Msg("couldn't get remote blob") diff --git a/pkg/extensions/sync/references/referrers_tag.go b/pkg/extensions/sync/references/referrers_tag.go index 21ffd2755..ffed36928 100644 --- a/pkg/extensions/sync/references/referrers_tag.go +++ b/pkg/extensions/sync/references/referrers_tag.go @@ -151,7 +151,7 @@ func (ref TagReferences) getIndex( ) (ispec.Index, []byte, error) { var index ispec.Index - content, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex, + content, _, statusCode, err := ref.client.MakeGetRequest(ctx, &index, ispec.MediaTypeImageIndex, "", "v2", repo, "manifests", getReferrersTagFromSubjectDigest(subjectDigestStr)) if err != nil { if statusCode == http.StatusNotFound { diff --git a/pkg/extensions/sync/remote.go b/pkg/extensions/sync/remote.go index fef205d42..bf85f62f3 100644 --- a/pkg/extensions/sync/remote.go +++ b/pkg/extensions/sync/remote.go @@ -6,6 +6,7 @@ package sync import ( "context" "fmt" + "net/url" "strings" "github.com/containers/image/v5/docker" @@ -50,13 +51,37 @@ func (registry *RemoteRegistry) GetContext() *types.SystemContext { func (registry *RemoteRegistry) GetRepositories(ctx context.Context) ([]string, error) { var catalog catalog - _, _, _, err := registry.client.MakeGetRequest(ctx, &catalog, "application/json", //nolint: dogsled + _, header, _, err := registry.client.MakeGetRequest(ctx, &catalog, "application/json", "", //nolint: dogsled constants.RoutePrefix, constants.ExtCatalogPrefix) if err != nil { return []string{}, err } - return catalog.Repositories, nil + var repos []string + + repos = append(repos, catalog.Repositories...) + + link := header.Get("Link") + for link != "" { + linkURLPart, _, _ := strings.Cut(link, ";") + + linkURL, err := url.Parse(strings.Trim(linkURLPart, "<>")) + if err != nil { + return catalog.Repositories, err + } + + _, header, _, err := registry.client.MakeGetRequest(ctx, &catalog, "application/json", + linkURL.RawQuery, constants.RoutePrefix, constants.ExtCatalogPrefix) //nolint: dogsled + if err != nil { + return repos, err + } + + repos = append(repos, catalog.Repositories...) + + link = header.Get("Link") + } + + return repos, nil } func (registry *RemoteRegistry) GetDockerRemoteRepo(repo string) string { diff --git a/pkg/storage/imagestore/imagestore.go b/pkg/storage/imagestore/imagestore.go index 7664aeb73..d37842e0e 100644 --- a/pkg/storage/imagestore/imagestore.go +++ b/pkg/storage/imagestore/imagestore.go @@ -265,6 +265,89 @@ func (is *ImageStore) ValidateRepo(name string) (bool, error) { return true, nil } +func (is *ImageStore) GetNextRepositories(lastRepo string, maxEntries int, filterFn storageTypes.FilterRepoFunc, +) ([]string, bool, error) { + var lockLatency time.Time + + dir := is.rootDir + + is.RLock(&lockLatency) + defer is.RUnlock(&lockLatency) + + stores := make([]string, 0) + + moreEntries := false + entries := 0 + found := false + err := is.storeDriver.Walk(dir, func(fileInfo driver.FileInfo) error { + if entries == maxEntries { + moreEntries = true + + return io.EOF + } + + if !fileInfo.IsDir() { + return nil + } + + // skip .sync and .uploads dirs no need to try to validate them + if strings.HasSuffix(fileInfo.Path(), syncConstants.SyncBlobUploadDir) || + strings.HasSuffix(fileInfo.Path(), ispec.ImageBlobsDir) || + strings.HasSuffix(fileInfo.Path(), storageConstants.BlobUploadDir) { + return driver.ErrSkipDir + } + + rel, err := filepath.Rel(is.rootDir, fileInfo.Path()) + if err != nil { + return nil //nolint:nilerr // ignore paths that are not under root dir + } + + if ok, err := is.ValidateRepo(rel); !ok || err != nil { + return nil //nolint:nilerr // ignore invalid repos + } + + if lastRepo == rel { + found = true + + return nil + } + + if lastRepo == "" { + found = true + } + + ok, err := filterFn(rel) + if err != nil { + return err + } + + if found && ok { + entries++ + + stores = append(stores, rel) + } + + return nil + }) + + // if the root directory is not yet created then return an empty slice of repositories + + driverErr := &driver.Error{} + + if errors.As(err, &driver.PathNotFoundError{}) { + is.log.Debug().Msg("empty rootDir") + + return stores, false, nil + } + + if errors.Is(err, io.EOF) || + (errors.As(err, driverErr) && errors.Is(driverErr.Detail, io.EOF)) { + return stores, moreEntries, nil + } + + return stores, moreEntries, err +} + // GetRepositories returns a list of all the repositories under this store. func (is *ImageStore) GetRepositories() ([]string, error) { var lockLatency time.Time diff --git a/pkg/storage/storage_controller.go b/pkg/storage/storage_controller.go index 721a1a60f..75f10f91b 100644 --- a/pkg/storage/storage_controller.go +++ b/pkg/storage/storage_controller.go @@ -7,8 +7,9 @@ import ( ) const ( - CosignType = "cosign" - NotationType = "notation" + CosignType = "cosign" + NotationType = "notation" + DefaultStorePath = "/" ) type StoreController struct { @@ -29,6 +30,21 @@ func GetRoutePrefix(name string) string { return "/" + names[0] } +func (sc StoreController) GetStorePath(name string) string { + if sc.SubStore != nil && name != "" { + subStorePath := GetRoutePrefix(name) + + _, ok := sc.SubStore[subStorePath] + if !ok { + return DefaultStorePath + } + + return subStorePath + } + + return DefaultStorePath +} + func (sc StoreController) GetImageStore(name string) storageTypes.ImageStore { if sc.SubStore != nil { // SubStore is being provided, now we need to find equivalent image store and this will be found by splitting name diff --git a/pkg/storage/storage_test.go b/pkg/storage/storage_test.go index 6ac19a4a8..5a88621fb 100644 --- a/pkg/storage/storage_test.go +++ b/pkg/storage/storage_test.go @@ -283,6 +283,14 @@ func TestStorageAPIs(t *testing.T) { repos, err := imgStore.GetRepositories() So(err, ShouldBeNil) So(repos, ShouldNotBeEmpty) + + repos, more, err := imgStore.GetNextRepositories("", -1, func(repo string) (bool, error) { + return true, nil + }) + + So(more, ShouldBeFalse) + So(err, ShouldBeNil) + So(repos, ShouldNotBeEmpty) }) Convey("Get image tags", func() { @@ -564,6 +572,21 @@ func TestStorageAPIs(t *testing.T) { So(len(repos), ShouldEqual, 1) So(repos[0], ShouldEqual, "test") + repos, more, err := imgStore.GetNextRepositories("", -1, func(repo string) (bool, error) { + return true, nil + }) + So(err, ShouldBeNil) + So(more, ShouldBeFalse) + So(len(repos), ShouldEqual, 1) + So(repos[0], ShouldEqual, "test") + + repos, more, err = imgStore.GetNextRepositories("", -1, func(repo string) (bool, error) { + return false, nil + }) + So(err, ShouldBeNil) + So(more, ShouldBeFalse) + So(len(repos), ShouldEqual, 0) + // We deleted only one tag, make sure blob should not be removed. hasBlob, _, err = imgStore.CheckBlob("test", digest) So(err, ShouldBeNil) diff --git a/pkg/storage/types/types.go b/pkg/storage/types/types.go index 9d5cd4882..edc067073 100644 --- a/pkg/storage/types/types.go +++ b/pkg/storage/types/types.go @@ -12,6 +12,8 @@ import ( "zotregistry.dev/zot/pkg/scheduler" ) +type FilterRepoFunc func(repo string) (bool, error) + type StoreController interface { GetImageStore(name string) ImageStore GetDefaultImageStore() ImageStore @@ -30,6 +32,7 @@ type ImageStore interface { //nolint:interfacebloat ValidateRepo(name string) (bool, error) GetRepositories() ([]string, error) GetNextRepository(repo string) (string, error) + GetNextRepositories(repo string, maxEntries int, fn FilterRepoFunc) ([]string, bool, error) GetImageTags(repo string) ([]string, error) GetImageManifest(repo, reference string) ([]byte, godigest.Digest, string, error) PutImageManifest(repo, reference, mediaType string, body []byte) (godigest.Digest, godigest.Digest, error) diff --git a/pkg/test/mocks/image_store_mock.go b/pkg/test/mocks/image_store_mock.go index d94b540d8..5d125a93d 100644 --- a/pkg/test/mocks/image_store_mock.go +++ b/pkg/test/mocks/image_store_mock.go @@ -9,19 +9,21 @@ import ( ispec "github.com/opencontainers/image-spec/specs-go/v1" "zotregistry.dev/zot/pkg/scheduler" + storageTypes "zotregistry.dev/zot/pkg/storage/types" ) type MockedImageStore struct { - NameFn func() string - DirExistsFn func(d string) bool - RootDirFn func() string - InitRepoFn func(name string) error - ValidateRepoFn func(name string) (bool, error) - GetRepositoriesFn func() ([]string, error) - GetNextRepositoryFn func(repo string) (string, error) - GetImageTagsFn func(repo string) ([]string, error) - GetImageManifestFn func(repo string, reference string) ([]byte, godigest.Digest, string, error) - PutImageManifestFn func(repo string, reference string, mediaType string, body []byte) (godigest.Digest, + NameFn func() string + DirExistsFn func(d string) bool + RootDirFn func() string + InitRepoFn func(name string) error + ValidateRepoFn func(name string) (bool, error) + GetRepositoriesFn func() ([]string, error) + GetNextRepositoryFn func(repo string) (string, error) + GetNextRepositoriesFn func(lastRepo string, maxEntries int, fn storageTypes.FilterRepoFunc) ([]string, bool, error) + GetImageTagsFn func(repo string) ([]string, error) + GetImageManifestFn func(repo string, reference string) ([]byte, godigest.Digest, string, error) + PutImageManifestFn func(repo string, reference string, mediaType string, body []byte) (godigest.Digest, godigest.Digest, error) DeleteImageManifestFn func(repo string, reference string, detectCollision bool) error BlobUploadPathFn func(repo string, uuid string) string @@ -138,6 +140,16 @@ func (is MockedImageStore) GetNextRepository(repo string) (string, error) { return "", nil } +func (is MockedImageStore) GetNextRepositories(lastRepo string, maxEntries int, + fn storageTypes.FilterRepoFunc, +) ([]string, bool, error) { + if is.GetNextRepositoriesFn != nil { + return is.GetNextRepositoriesFn(lastRepo, maxEntries, fn) + } + + return []string{}, false, nil +} + func (is MockedImageStore) GetImageManifest(repo string, reference string) ([]byte, godigest.Digest, string, error) { if is.GetImageManifestFn != nil { return is.GetImageManifestFn(repo, reference) diff --git a/test/blackbox/pushpull.bats b/test/blackbox/pushpull.bats index c1ea075a0..5edaeed7c 100644 --- a/test/blackbox/pushpull.bats +++ b/test/blackbox/pushpull.bats @@ -248,6 +248,19 @@ function teardown_file() { fi done [ "$found" -eq 1 ] + + run regctl repo ls --limit 2 localhost:${zot_port} + [ "$status" -eq 0 ] + echo "$output" + [ $(echo "$output" | wc -l) -eq 2 ] + [ "${lines[-2]}" == "busybox" ] + [ "${lines[-1]}" == "golang" ] + + run regctl repo ls --last busybox --limit 1 localhost:${zot_port} + [ "$status" -eq 0 ] + echo "$output" + [ $(echo "$output" | wc -l) -eq 1 ] + [ "${lines[-1]}" == "golang" ] } @test "list image tags with regclient" {