From a5f74e860e1768d74f68e730b92c384b741eb27a Mon Sep 17 00:00:00 2001 From: jay-dee7 Date: Wed, 8 Nov 2023 21:05:27 +0530 Subject: [PATCH] WIP Signed-off-by: jay-dee7 --- auth/github.go | 1 + auth/invites.go | 4 + auth/jwt_middleware.go | 4 +- auth/reset_password.go | 7 + auth/server/webauthn_server.go | 5 + auth/sessions.go | 1 - cmd/extras/digest.go | 16 +- conformance.vars | 1 - dfs/filebase/filebase.go | 6 +- dfs/storj/storj.go | 6 +- go.mod | 1 + go.sum | 4 + registry/v2/blobs.go | 10 +- registry/v2/extensions/catalog_detail.go | 4 + registry/v2/registry.go | 34 ++-- registry/v2/types.go | 91 ---------- store/v2/registry/registry_impl.go | 210 ++++++++++++++++++++--- store/v2/registry/store.go | 3 +- store/v2/types/registry.go | 43 ++--- telemetry/log.go | 3 +- 20 files changed, 272 insertions(+), 182 deletions(-) diff --git a/auth/github.go b/auth/github.go index c5f31bc7..e53ec8a3 100644 --- a/auth/github.go +++ b/auth/github.go @@ -22,6 +22,7 @@ import ( func (a *auth) LoginWithGithub(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) + state, err := uuid.NewRandom() if err != nil { echoErr := ctx.JSON(http.StatusBadRequest, echo.Map{ diff --git a/auth/invites.go b/auth/invites.go index 2c48e4c0..d9469992 100644 --- a/auth/invites.go +++ b/auth/invites.go @@ -6,7 +6,9 @@ import ( "net/http" "regexp" "strings" + "time" + "github.com/containerish/OpenRegistry/types" "github.com/labstack/echo/v4" ) @@ -15,6 +17,8 @@ type List struct { } func (a *auth) Invites(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) + var list List err := json.NewDecoder(ctx.Request().Body).Decode(&list) if err != nil { diff --git a/auth/jwt_middleware.go b/auth/jwt_middleware.go index 21d0f7b0..5cace51d 100644 --- a/auth/jwt_middleware.go +++ b/auth/jwt_middleware.go @@ -23,6 +23,8 @@ const ( func (a *auth) JWT() echo.MiddlewareFunc { return echo_jwt.WithConfig(echo_jwt.Config{ Skipper: func(ctx echo.Context) bool { + ctx.Set(types.HandlerStartTime, time.Now()) + if strings.HasPrefix(ctx.Request().RequestURI, "/auth") { return false } @@ -46,7 +48,6 @@ func (a *auth) JWT() echo.MiddlewareFunc { return true }, ErrorHandler: func(ctx echo.Context, err error) error { - ctx.Set(types.HandlerStartTime, time.Now()) echoErr := ctx.JSON(http.StatusUnauthorized, echo.Map{ "error": err.Error(), "message": "missing authentication information", @@ -131,6 +132,7 @@ func (a *auth) JWTRest() echo.MiddlewareFunc { return echo_jwt.WithConfig(echo_jwt.Config{ ErrorHandler: func(ctx echo.Context, err error) error { ctx.Set(types.HandlerStartTime, time.Now()) + echoErr := ctx.JSON(http.StatusUnauthorized, echo.Map{ "error": err.Error(), "message": "missing authentication information", diff --git a/auth/reset_password.go b/auth/reset_password.go index db3e9216..6e9801de 100644 --- a/auth/reset_password.go +++ b/auth/reset_password.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "time" "github.com/containerish/OpenRegistry/services/email" v2_types "github.com/containerish/OpenRegistry/store/v2/types" @@ -16,6 +17,8 @@ import ( ) func (a *auth) ResetForgottenPassword(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) + token, ok := ctx.Get("user").(*jwt.Token) if !ok { err := fmt.Errorf("ERR_EMPTY_TOKEN") @@ -108,6 +111,8 @@ and an uppercase letter`, } func (a *auth) ResetPassword(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) + token, ok := ctx.Get("user").(*jwt.Token) if !ok { err := fmt.Errorf("ERR_EMPTY_TOKEN") @@ -212,6 +217,8 @@ and an uppercase letter`, } func (a *auth) ForgotPassword(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) + userEmail := ctx.QueryParam("email") if err := a.verifyEmail(userEmail); err != nil { echoErr := ctx.JSON(http.StatusBadRequest, echo.Map{ diff --git a/auth/server/webauthn_server.go b/auth/server/webauthn_server.go index 9a579765..e14a89f1 100644 --- a/auth/server/webauthn_server.go +++ b/auth/server/webauthn_server.go @@ -85,6 +85,7 @@ func (wa *webauthn_server) webAuthNTxnCleanup() { func (wa *webauthn_server) BeginRegistration(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) + user := v2_types.User{} if err := json.NewDecoder(ctx.Request().Body).Decode(&user); err != nil { @@ -195,6 +196,8 @@ func (wa *webauthn_server) BeginRegistration(ctx echo.Context) error { } func (wa *webauthn_server) RollbackRegistration(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) + username := ctx.QueryParam("username") meta, ok := wa.txnStore[username] if !ok { @@ -226,6 +229,8 @@ func (wa *webauthn_server) RollbackRegistration(ctx echo.Context) error { } func (wa *webauthn_server) RollbackSessionData(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) + username := ctx.QueryParam("username") if username == "" { return ctx.JSON(http.StatusBadRequest, echo.Map{ diff --git a/auth/sessions.go b/auth/sessions.go index 9cee9072..e56dc702 100644 --- a/auth/sessions.go +++ b/auth/sessions.go @@ -14,7 +14,6 @@ import ( ) func (a *auth) ExpireSessions(ctx echo.Context) error { - //queryParamSessionId := ctx.QueryParam("session_id") ctx.Set(types.HandlerStartTime, time.Now()) sessionCookie, err := ctx.Cookie("session_id") diff --git a/cmd/extras/digest.go b/cmd/extras/digest.go index d115e927..95b6958f 100644 --- a/cmd/extras/digest.go +++ b/cmd/extras/digest.go @@ -6,7 +6,7 @@ import ( "strings" "github.com/fatih/color" - "github.com/opencontainers/go-digest" + oci_digest "github.com/opencontainers/go-digest" "github.com/urfave/cli/v2" ) @@ -45,21 +45,23 @@ func generateDigest(ctx *cli.Context) error { digestType = "CANONICAL" } - manifestContent, err := json.MarshalIndent(input, "", "\t") + // inputBz := bytes.TrimSpace([]byte(input)) + + manifestContent, err := json.Marshal([]byte(input)) if err != nil { return fmt.Errorf(color.RedString("generateDigest: ERR_MARSHAL_INDENT: %s", err)) } - var inputDigest digest.Digest + var inputDigest oci_digest.Digest switch digestType { case "SHA256": - inputDigest = digest.SHA256.FromBytes(manifestContent) + inputDigest = oci_digest.SHA256.FromBytes(manifestContent) case "SHA512": - inputDigest = digest.SHA512.FromBytes(manifestContent) + inputDigest = oci_digest.SHA512.FromBytes(manifestContent) default: - inputDigest = digest.Canonical.FromBytes(manifestContent) + inputDigest = oci_digest.Canonical.FromBytes(manifestContent) } - color.Green("%s", inputDigest) + color.Green("input=\n%s\ndigest=%s", manifestContent, inputDigest) return nil } diff --git a/conformance.vars b/conformance.vars index 04900ddd..a5d35152 100644 --- a/conformance.vars +++ b/conformance.vars @@ -7,4 +7,3 @@ OCI_PASSWORD=Qwerty@123 OCI_ROOT_URL=http://localhost:5000 OCI_NAMESPACE=johndoe/dummmy-server OCI_CROSSMOUNT_NAMESPACE=johndoe/dummmy-server-cross-mount -OCI_DELETE_MANIFEST_BEFORE_BLOBS=0 diff --git a/dfs/filebase/filebase.go b/dfs/filebase/filebase.go index c5540748..36439267 100644 --- a/dfs/filebase/filebase.go +++ b/dfs/filebase/filebase.go @@ -13,7 +13,7 @@ import ( "github.com/containerish/OpenRegistry/config" "github.com/containerish/OpenRegistry/dfs" "github.com/containerish/OpenRegistry/store/v2/types" - "github.com/opencontainers/go-digest" + oci_digest "github.com/opencontainers/go-digest" ) type filebase struct { @@ -119,7 +119,7 @@ func (fb *filebase) CompleteMultipartUpload( ctx, cancel := context.WithTimeout(ctx, time.Minute*5) defer cancel() - dig, err := digest.Parse(layerDigest) + digest, err := oci_digest.Parse(layerDigest) if err != nil { return "", fmt.Errorf("ERR_FILEBASE_DIGEST_PARSE: %w", err) } @@ -128,7 +128,7 @@ func (fb *filebase) CompleteMultipartUpload( Key: &layerKey, Bucket: &fb.bucket, UploadId: &uploadId, - ChecksumSHA256: aws.String(dig.Encoded()), + ChecksumSHA256: aws.String(digest.Encoded()), MultipartUpload: &s3types.CompletedMultipartUpload{Parts: completedParts}, }) if err != nil { diff --git a/dfs/storj/storj.go b/dfs/storj/storj.go index b08cc521..b347a221 100644 --- a/dfs/storj/storj.go +++ b/dfs/storj/storj.go @@ -13,7 +13,7 @@ import ( "github.com/containerish/OpenRegistry/config" "github.com/containerish/OpenRegistry/dfs" "github.com/containerish/OpenRegistry/store/v2/types" - "github.com/opencontainers/go-digest" + oci_digest "github.com/opencontainers/go-digest" ) type storj struct { @@ -103,7 +103,7 @@ func (sj *storj) CompleteMultipartUpload( ctx, cancel := context.WithTimeout(ctx, time.Minute*5) defer cancel() - dig, err := digest.Parse(layerDigest) + digest, err := oci_digest.Parse(layerDigest) if err != nil { return "", fmt.Errorf("ERR_STORJ_DIGEST_PARSE: %w", err) } @@ -112,7 +112,7 @@ func (sj *storj) CompleteMultipartUpload( Key: &layerKey, Bucket: &sj.bucket, UploadId: &uploadId, - ChecksumSHA256: aws.String(dig.Encoded()), + ChecksumSHA256: aws.String(digest.Encoded()), MultipartUpload: &s3types.CompletedMultipartUpload{Parts: completedParts}, }) if err != nil { diff --git a/go.mod b/go.mod index d5213b26..63b6747f 100644 --- a/go.mod +++ b/go.mod @@ -43,6 +43,7 @@ require ( golang.org/x/oauth2 v0.13.0 gopkg.in/yaml.v2 v2.4.0 storj.io/uplink v1.12.1 + github.com/opencontainers/image-spec v1.1.0-rc5 ) require ( diff --git a/go.sum b/go.sum index e69d1777..5fb5d3a6 100644 --- a/go.sum +++ b/go.sum @@ -412,6 +412,10 @@ github.com/opencontainers/distribution-spec v1.0.1 h1:hT6tF6uKZAQh+HH3BrJqn7/xHh github.com/opencontainers/distribution-spec v1.0.1/go.mod h1:copR2flp+jTEvQIFMb6MIx45OkrxzqyjszPDT3hx/5Q= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= +github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= +github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/registry/v2/blobs.go b/registry/v2/blobs.go index 952e077b..77af2c3b 100644 --- a/registry/v2/blobs.go +++ b/registry/v2/blobs.go @@ -12,7 +12,7 @@ import ( "github.com/containerish/OpenRegistry/types" "github.com/fatih/color" "github.com/labstack/echo/v4" - "github.com/opencontainers/go-digest" + oci_digest "github.com/opencontainers/go-digest" ) func (b *blobs) errorResponse(code, msg string, detail map[string]interface{}) []byte { @@ -103,14 +103,14 @@ func (b *blobs) UploadBlob(ctx echo.Context) error { } defer ctx.Request().Body.Close() - checksum := digest.FromBytes(buf.Bytes()) + digest := oci_digest.FromBytes(buf.Bytes()) b.blobCounter[uploadID]++ part, err := b.registry.dfs.UploadPart( ctx.Request().Context(), uploadID, GetLayerIdentifier(layerKey), - checksum.String(), + digest.String(), b.blobCounter[uploadID], bytes.NewReader(buf.Bytes()), int64(buf.Len()), @@ -171,13 +171,13 @@ func (b *blobs) UploadBlob(ctx echo.Context) error { } defer ctx.Request().Body.Close() - checksum := digest.FromBytes(buf.Bytes()) + digest := oci_digest.FromBytes(buf.Bytes()) b.blobCounter[uploadID]++ part, err := b.registry.dfs.UploadPart( ctx.Request().Context(), uploadID, GetLayerIdentifier(layerKey), - checksum.String(), + digest.String(), b.blobCounter[uploadID], bytes.NewReader(buf.Bytes()), int64(buf.Len()), diff --git a/registry/v2/extensions/catalog_detail.go b/registry/v2/extensions/catalog_detail.go index 78dbb56f..b3727390 100644 --- a/registry/v2/extensions/catalog_detail.go +++ b/registry/v2/extensions/catalog_detail.go @@ -164,6 +164,8 @@ func (ext *extension) RepositoryDetail(ctx echo.Context) error { } func (ext *extension) PublicCatalog(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) + queryParamPageSize := ctx.QueryParam("n") queryParamOffset := ctx.QueryParam("last") var pageSize int @@ -206,6 +208,8 @@ func (ext *extension) PublicCatalog(ctx echo.Context) error { } func (ext *extension) GetUserCatalog(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) + user, ok := ctx.Get(string(types.UserContextKey)).(*types.User) if !ok { errMsg := fmt.Errorf("missing user in request context") diff --git a/registry/v2/registry.go b/registry/v2/registry.go index b2ce7d31..f0180ed0 100644 --- a/registry/v2/registry.go +++ b/registry/v2/registry.go @@ -23,6 +23,7 @@ import ( "github.com/containerish/OpenRegistry/types" "github.com/labstack/echo/v4" oci_digest "github.com/opencontainers/go-digest" + img_spec_v1 "github.com/opencontainers/image-spec/specs-go/v1" ) func NewRegistry( @@ -205,6 +206,7 @@ func (r *registry) List(ctx echo.Context) error { // GET /v2//manifests/ func (r *registry) PullManifest(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) + namespace := ctx.Get(string(RegistryNamespace)).(string) ref := ctx.Param("reference") @@ -698,6 +700,7 @@ func (r *registry) PushImage(ctx echo.Context) error { // Path: /v2//manifests/ func (r *registry) PushManifest(ctx echo.Context) error { ctx.Set(types.HandlerStartTime, time.Now()) + namespace := ctx.Get(string(RegistryNamespace)).(string) ref := ctx.Param("reference") @@ -753,6 +756,7 @@ func (r *registry) PushManifest(ctx echo.Context) error { }) } defer ctx.Request().Body.Close() + digest := oci_digest.FromBytes(buf.Bytes()) uuid, err := NewUUID() @@ -783,13 +787,13 @@ func (r *registry) PushManifest(ctx echo.Context) error { } if manifest.MediaType == "" { - manifest.MediaType = OCIMediaTypeV1ImageManifest.String() + manifest.MediaType = img_spec_v1.MediaTypeImageManifest } var layerIDs []string for _, layer := range manifest.Layers { - layerIDs = append(layerIDs, layer.Digest) + layerIDs = append(layerIDs, layer.Digest.String()) } size, err := r.store.GetImageSizeByLayerIds(ctx.Request().Context(), layerIDs) @@ -845,8 +849,8 @@ func (r *registry) setPushManifestHaeders( ctx.Response().Header().Set("Content-Type", manifest.MediaType) // set OCI-Subject header if we get a manifest with a subject - if manifest.Subject != nil && manifest.Subject.Digest != "" { - ctx.Response().Header().Set("OCI-Subject", manifest.Subject.Digest) + if manifest.Subject != nil { + ctx.Response().Header().Set("OCI-Subject", manifest.Subject.Digest.String()) } } @@ -945,11 +949,16 @@ func (r *registry) DeleteLayer(ctx echo.Context) error { // Should also look into 401 Code // https://docs.docker.com/registry/spec/api/ func (r *registry) ApiVersion(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) + ctx.Response().Header().Set(HeaderDockerDistributionApiVersion, "registry/2.0") - return ctx.String(http.StatusOK, "OK\n") + echoErr := ctx.String(http.StatusOK, "OK\n") + r.logger.Log(ctx, nil).Send() + return echoErr } func (r *registry) GetImageNamespace(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) searchQuery := ctx.QueryParam("search_query") if searchQuery == "" { @@ -997,6 +1006,7 @@ func (r *registry) GetRepositoryFromCtx(ctx echo.Context) *types_v2.ContainerIma } func (r *registry) ListReferrers(ctx echo.Context) error { + ctx.Set(types.HandlerStartTime, time.Now()) ctx.Set("start", time.Now()) namespace := ctx.Get(string(RegistryNamespace)).(string) @@ -1019,19 +1029,14 @@ func (r *registry) ListReferrers(ctx echo.Context) error { filterValues = append(filterValues, filter) } } - ctx.Response().Header().Set("OCI-Filters-Applied", OCIFilterArtifactType.String()) - } - - refIndex := types_v2.ReferrerImageIndex{ - SchemaVersion: 2, - MediaType: OCIMediaTypeV1ImageIndex.String(), + ctx.Response().Header().Set("OCI-Filters-Applied", "artifactType") } - refIndex.Manifests, err = r.store.GetReferrers(ctx.Request().Context(), namespace, digest, filterValues) + ctx.Response().Header().Set("content-type", img_spec_v1.MediaTypeImageIndex) + refIndex, err := r.store.GetReferrers(ctx.Request().Context(), namespace, digest, filterValues) if err != nil { - ctx.Response().Header().Set("content-type", OCIMediaTypeV1ImageIndex.String()) echoErr := ctx.JSON(http.StatusOK, refIndex) - r.logger.Log(ctx, nil). + r.logger.Log(ctx, err). Bool("referrersFound", false). Str("artifactType", artifactType). Str("digest", digest). @@ -1039,7 +1044,6 @@ func (r *registry) ListReferrers(ctx echo.Context) error { return echoErr } - ctx.Response().Header().Set("content-type", OCIMediaTypeV1ImageIndex.String()) echoErr := ctx.JSON(http.StatusOK, refIndex) r.logger.Log(ctx, nil). Bool("referrersFound", true). diff --git a/registry/v2/types.go b/registry/v2/types.go index 70ee6d97..de891b9a 100644 --- a/registry/v2/types.go +++ b/registry/v2/types.go @@ -251,94 +251,3 @@ type Registry interface { ListReferrers(ctx echo.Context) error } - -// reference: https://github.com/opencontainers/image-spec/blob/main/media-types.md#oci-image-media-types -type OCIMediaTypeV1 int -type DockerDistributionMediaType OCIMediaTypeV1 - -const ( - // application/vnd.oci.descriptor.v1+json - OCIMediaTypeV1ContentDescriptor OCIMediaTypeV1 = iota + 1 - // application/vnd.oci.layout.header.v1+json - OCIMediaTypeV1LayoutHeader - // application/vnd.oci.image.index.v1+json - OCIMediaTypeV1ImageIndex - // application/vnd.oci.image.manifest.v1+json - OCIMediaTypeV1ImageManifest - // application/vnd.oci.image.config.v1+json - OCIMediaTypeV1ImageConfig - // application/vnd.oci.image.layer.v1.tar - OCIMediaTypeV1LayerTar - // application/vnd.oci.image.layer.v1.tar+gzip - OCIMediaTypeV1LayerGzip - // application/vnd.oci.image.layer.v1.tar+zstd - OCIMediaTypeV1LayerZStd - // application/vnd.oci.empty.v1+json - OCIMediaTypeV1Empty -) - -const ( - // application/vnd.docker.distribution.manifest.list.v2+json - DistributionV2ManifestList DockerDistributionMediaType = iota + 1 - // application/vnd.docker.distribution.manifest.v2+json - DistributionV2Manifest - // application/vnd.docker.image.rootfs.diff.tar.gzip - DistributionV2ImageRootFS - // application/vnd.docker.container.image.v1+json - DistributionV1Image -) - -func (m OCIMediaTypeV1) String() string { - switch m { - case OCIMediaTypeV1ContentDescriptor: - return "application/vnd.oci.descriptor.v1+json" - case OCIMediaTypeV1LayoutHeader: - return "application/vnd.oci.layout.header.v1+json" - case OCIMediaTypeV1ImageIndex: - return "application/vnd.oci.image.index.v1+json" - case OCIMediaTypeV1ImageManifest: - return "application/vnd.oci.image.manifest.v1+json" - case OCIMediaTypeV1ImageConfig: - return "application/vnd.oci.image.config.v1+json" - case OCIMediaTypeV1LayerTar: - return "application/vnd.oci.image.layer.v1.tar" - case OCIMediaTypeV1LayerGzip: - return "application/vnd.oci.image.layer.v1.tar+gzip" - case OCIMediaTypeV1LayerZStd: - return "application/vnd.oci.image.layer.v1.tar+zstd" - case OCIMediaTypeV1Empty: - return "application/vnd.oci.empty.v1+json" - default: - return "application/vnd.oci.empty.v1+json" - } -} - -func (m DockerDistributionMediaType) String() string { - switch m { - case DistributionV2ManifestList: - return "application/vnd.docker.distribution.manifest.list.v2+json" - case DistributionV2Manifest: - return "application/vnd.docker.distribution.manifest.v2+json" - case DistributionV2ImageRootFS: - return "application/vnd.docker.image.rootfs.diff.tar.gzip" - case DistributionV1Image: - return "application/vnd.docker.container.image.v1+json" - default: - return "application/vnd.oci.empty.v1+json" - } -} - -type OCIFilter int - -const ( - OCIFilterArtifactType OCIFilter = iota + 1 -) - -func (f OCIFilter) String() string { - switch f { - case OCIFilterArtifactType: - return "artifactType" - default: - return "" - } -} diff --git a/store/v2/registry/registry_impl.go b/store/v2/registry/registry_impl.go index ca001cf1..8741bdc8 100644 --- a/store/v2/registry/registry_impl.go +++ b/store/v2/registry/registry_impl.go @@ -9,7 +9,11 @@ import ( v2 "github.com/containerish/OpenRegistry/store/v2" "github.com/containerish/OpenRegistry/store/v2/types" + "github.com/fatih/color" "github.com/google/uuid" + oci_digest "github.com/opencontainers/go-digest" + img_spec "github.com/opencontainers/image-spec/specs-go" + img_spec_v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/uptrace/bun" "github.com/uptrace/bun/dialect/feature" ) @@ -565,51 +569,201 @@ func (s *registryStore) GetReferrers( namespace string, digest string, artifactTypes []string, -) ([]*types.ImageManifestSubject, error) { - var manifestList []*types.ImageManifest +) (*img_spec_v1.Index, error) { + var manifests []*types.ImageManifest // we return an empty list on error - subjectList := []*types.ImageManifestSubject{} + imgIndex := &img_spec_v1.Index{ + Versioned: img_spec.Versioned{ + SchemaVersion: 2, + }, + MediaType: img_spec_v1.MediaTypeImageIndex, + } + nsParts := strings.Split(namespace, "/") + if len(nsParts) != 2 { + return imgIndex, fmt.Errorf("GetReferrers: invalid namespace format") + } + + username, repoName := nsParts[0], nsParts[1] q := s. db. NewSelect(). - Model(&manifestList) + Model(&manifests). + WhereOr("COALESCE(subject_digest, '') = '' AND digest = ?", digest). + WhereOr("subject_digest = ?", digest). + Relation("Repository", func(sq *bun.SelectQuery) *bun.SelectQuery { + return sq.ExcludeColumn("*").Where("name = ?", repoName) + }). + Relation("User", func(sq *bun.SelectQuery) *bun.SelectQuery { + return sq.ExcludeColumn("*").Where("username = ?", username) + }) if len(artifactTypes) > 0 { - q.Where("subject_digest = ? AND artifact_type in (?)", digest, bun.In(artifactTypes)) - } else { - q.Where("subject_digest = ?", digest) + q. + WhereOr("artifact_type IN (?)", bun.In(artifactTypes)) + // WhereOr("artifact_type = '' AND config_artifact_type IN (?)", bun.In(artifactTypes)) } - nsParts := strings.Split(namespace, "/") - if len(nsParts) != 2 { - return subjectList, fmt.Errorf("GetReferrers: invalid namespace format") - } - - username, repoName := nsParts[0], nsParts[1] - q.Relation("Repository", func(sq *bun.SelectQuery) *bun.SelectQuery { - return sq.ExcludeColumn("*").Where("name = ?", repoName) - }).Relation("User", func(sq *bun.SelectQuery) *bun.SelectQuery { - return sq.Where("username = ?", username).ExcludeColumn("*") - }) + color.Magenta("query: %s", q.String()) if err := q.Scan(ctx); err != nil { - return subjectList, err - } + return imgIndex, nil + } + + // q := s. + // db. + // NewSelect(). + // Model(&manifestList) + // + // // Find OCI Distribution Spec < 1.1 manifests to provide backword compatibility + // // Reference - https://github.com/opencontainers/distribution-spec/blob/main/spec.md#enabling-the-referrers-api + // q.WhereGroup(" OR ", func(sq *bun.SelectQuery) *bun.SelectQuery { + // return sq. + // // for all new manifests with subject (OCI v1.1+), we set subject digest and manifest artifactType values + // Where("subject_digest = '' AND artifact_type = '' AND digest = ?", digest) + // }).WhereGroup(" OR ", func(sq *bun.SelectQuery) *bun.SelectQuery { + // return sq.Where("subject_digest = ?", digest) + // }) + // + // if len(artifactTypes) > 0 { + // q.WhereGroup(" AND ", func(sq *bun.SelectQuery) *bun.SelectQuery { + // return sq.Where("artifact_type in (?) or artifact_type = ''", bun.In(artifactTypes)) + // }) + // } + + // color.Magenta("GetReferrers: oldManifestsQuery: %s", oldMfQ.String()) + // color.Magenta("GetReferrers: newManifestsQuery: %s", newMfQ.String()) + // q.Relation("Repository", func(sq *bun.SelectQuery) *bun.SelectQuery { + // return sq.ExcludeColumn("*").Where("name = ?", repoName) + // }).Relation("User", func(sq *bun.SelectQuery) *bun.SelectQuery { + // return sq.Where("username = ?", username).ExcludeColumn("*") + // }) + + // eChan := make(chan error) + // go func() { + // wg.Wait() + // close(eChan) + // }() + // + // go func() { + // defer wg.Done() + // if err := oldMfQ.Scan(ctx); err != nil { + // eChan <- err + // return + // } + // }() + // + // go func() { + // defer wg.Done() + // if err := newMfQ.Scan(ctx); err != nil { + // eChan <- err + // return + // } + // }() + // + // mErr := &multierror.Error{} + // for err := range eChan { + // if err != nil { + // mErr.Errors = append(mErr.Errors, err) + // } + // } + // if mErr.ErrorOrNil() != nil { + // return imgIndex, mErr + // } + + // obz, _ := json.MarshalIndent(oldManifests, "", "\t") + // nbz, _ := json.MarshalIndent(manifests, "", "\t") + // color.Blue("OldManifests: \n%s\nNewManifests: \n%s", obz, nbz) + + // seenMap := map[string]struct{}{} + // for _, m := range append(oldManifests, newManifests...) { + // + // mapKey := fmt.Sprintf("%s_%s", m.MediaType, m.Digest) + // + // if m.Subject != nil { + // mapKey = fmt.Sprintf("%s_%s_%s", m.MediaType, m.Digest, m.Subject.MediaType) + // } else if m.Config != nil { + // mapKey = fmt.Sprintf("%s_%s", m.Config.MediaType, m.Config.Digest) + // } + // if _, ok := seenMap[mapKey]; ok { + // continue + // } + // if m.Subject != nil { + // seenMap[mapKey] = struct{}{} + // subjectList = append(subjectList, m.Subject) + // continue + // } + // + // seenMap[mapKey] = struct{}{} + // subjectList = append(subjectList, &types.OCIDescriptor{ + // MediaType: m.MediaType, + // Digest: oci_digest.FromString(m.Digest), + // Size: int64(m.Size), + // }) + // } + + // combinedManifests := append(oldManifests, manifests...) + var descriptors []img_spec_v1.Descriptor + + for _, m := range manifests { + d, err := oci_digest.Parse(m.Digest) + if err != nil { + continue + } - seenMap := map[string]struct{}{} - for _, m := range manifestList { - if m.Subject != nil { - if _, ok := seenMap[m.Subject.Digest]; ok { - continue + if !s.descriptorFound(descriptors, d.String()) { + if m.Subject != nil { + descriptors = append(descriptors, img_spec_v1.Descriptor{ + MediaType: m.MediaType, + Digest: d, + Size: int64(m.Size), + ArtifactType: m.ArtifactType, + }) + } else { + artifactType := m.ArtifactType + if artifactType == "" { + artifactType = m.Config.ArtifactType + } + + descriptors = append(descriptors, img_spec_v1.Descriptor{ + MediaType: m.MediaType, + Digest: d, + Size: int64(m.Size), + ArtifactType: artifactType, + }) } - seenMap[m.Subject.Digest] = struct{}{} - subjectList = append(subjectList, m.Subject) + } + + // switch m.MediaType { + // case img_spec_v1.MediaTypeImageManifest: + // case img_spec_v1.MediaTypeImageIndex: + // descriptors = append(descriptors, img_spec_v1.Descriptor{ + // Annotations: m.Subject.Annotations, + // MediaType: m.MediaType, + // Digest: oci_digest.FromString(m.Digest), + // Size: int64(m.Size), + // ArtifactType: m.ArtifactType, + // }) + // default: + // color.Red("----------------------------- FOUND_INVALID_MEDIA_TYPE -------------------------------------") + // } + } + + imgIndex.Manifests = descriptors + return imgIndex, nil +} + +func (s *registryStore) descriptorFound(descriptors []img_spec_v1.Descriptor, digest string) bool { + color.Yellow("total descriptors: %d", len(descriptors)) + for _, d := range descriptors { + color.Yellow("d.digest=%s - digest=%s - matched: %v", d.Digest.String(), digest, d.Digest.String() == digest) + if d.Digest.String() == digest { + return true } } - return subjectList, nil + return false } func (s *registryStore) GetImageSizeByLayerIds(ctx context.Context, layerIDs []string) (uint64, error) { diff --git a/store/v2/registry/store.go b/store/v2/registry/store.go index b3e1a442..c55f76f9 100644 --- a/store/v2/registry/store.go +++ b/store/v2/registry/store.go @@ -7,6 +7,7 @@ import ( "github.com/containerish/OpenRegistry/store/v2/types" "github.com/containerish/OpenRegistry/telemetry" "github.com/google/uuid" + img_spec_v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/uptrace/bun" ) @@ -38,7 +39,7 @@ type RegistryBaseStore interface { ns string, digest string, artifactTypes []string, - ) ([]*types.ImageManifestSubject, error) + ) (*img_spec_v1.Index, error) } type RegistryStore interface { diff --git a/store/v2/types/registry.go b/store/v2/types/registry.go index 4e8e583f..212d4323 100644 --- a/store/v2/types/registry.go +++ b/store/v2/types/registry.go @@ -10,6 +10,7 @@ import ( "github.com/fatih/color" "github.com/google/uuid" + img_spec_v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/uptrace/bun" ) @@ -42,8 +43,8 @@ type ( UpdatedAt time.Time `bun:"updated_at,nullzero" json:"updatedAt"` Repository *ContainerImageRepository `bun:"rel:belongs-to,join:repository_id=id" json:"-"` User *User `bun:"rel:belongs-to,join:owner_id=id" json:"-"` - Subject *ImageManifestSubject `bun:"embed:subject_" json:"subject,omitempty"` - Config *ImageManifestConfig `bun:"embed:config_" json:"config"` + Subject *img_spec_v1.Descriptor `bun:"embed:subject_" json:"subject,omitempty"` + Config *img_spec_v1.Descriptor `bun:"embed:config_" json:"config"` Reference string `bun:"reference,notnull" json:"reference"` Digest string `bun:"digest,notnull" json:"digest"` MediaType string `bun:"media_type,notnull" json:"mediaType"` @@ -56,28 +57,26 @@ type ( OwnerID uuid.UUID `bun:"owner_id,type:uuid" json:"ownerId"` } - ImageManifestSubject struct { - Annotations map[string]string `bun:"type:jsonb,nullzero" json:"annotations,omitempty"` - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - ArtifactType string `json:"artifactType,omitempty"` - NewUnspecifiedField string `json:"newUnspecifiedField,omitempty"` - Size uint64 `json:"size"` - } + // ImageManifestSubject struct { + // NewUnspecifiedField string `json:"newUnspecifiedField,omitempty"` + // OCIDescriptor + // } - ImageManifestConfig struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - Size uint64 `json:"size"` + Platform struct { + Architecture string `json:"architecture,omitempty"` + Variant string `json:"variant,omitempty"` + OS string `json:"os,omitempty"` + OSVersion string `json:"os.version,omitempty"` + OSFeatures []string `json:"os.features,omitempty"` } - ImageManifestLayer struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - Size uint64 `json:"size"` + ImageManifestPlatformOS struct { + Name string `json:"name"` + Version string `json:"version"` + Features []string `json:"features"` } - ImageManifestLayers []*ImageManifestLayer + ImageManifestLayers []*img_spec_v1.Descriptor ContainerImageLayer struct { bun.BaseModel `bun:"table:layers,alias:l" json:"-"` @@ -109,12 +108,6 @@ type ( } RepositoryVisibility string - - ReferrerImageIndex struct { - MediaType string `json:"mediaType"` - Manifests []*ImageManifestSubject `json:"manifests"` - SchemaVersion int `json:"schemaVersion"` - } ) var _ driver.Valuer = (*ImageManifestLayers)(nil) diff --git a/telemetry/log.go b/telemetry/log.go index 1d9cada6..2144abba 100644 --- a/telemetry/log.go +++ b/telemetry/log.go @@ -8,6 +8,7 @@ import ( "github.com/axiomhq/axiom-go/axiom" "github.com/containerish/OpenRegistry/config" + "github.com/containerish/OpenRegistry/types" "github.com/fatih/color" "github.com/labstack/echo/v4" "github.com/rs/zerolog" @@ -89,7 +90,7 @@ func setupLogger(config config.Logging) zerolog.Logger { func (l *logger) Log(ctx echo.Context, errMsg error) *zerolog.Event { stop := time.Now() - start, ok := ctx.Get("start").(time.Time) + start, ok := ctx.Get(types.HandlerStartTime).(time.Time) if !ok { start = stop }