add artifact_type for artifact model to support artifactType filter

Signed-off-by: yminer <yminer@vmware.com> add 2.11 sql schema & update index artifactType omitted Signed-off-by: yminer <yminer@vmware.com> update UT
goharbor · Feb 29, 2024 · 5694f87 · 5694f87
1 parent d79e4b1
commit 5694f87
Show file tree

Hide file tree

Showing 12 changed files with 342 additions and 21 deletions.
diff --git a/make/migrations/postgresql/0140_2.11.0_schema.up.sql b/make/migrations/postgresql/0140_2.11.0_schema.up.sql
@@ -0,0 +1,32 @@
+/*
+table artifact:
+  id            SERIAL PRIMARY KEY NOT NULL,
+  type          varchar(255) NOT NULL,
+  media_type    varchar(255) NOT NULL,
+  manifest_media_type varchar(255) NOT NULL,
+  artifact_type varchar(255) NOT NULL,
+  project_id    int NOT NULL,
+  repository_id int NOT NULL,
+  repository_name varchar(255) NOT NULL,
+  digest        varchar(255) NOT NULL,
+  size          bigint,
+  push_time     timestamp default CURRENT_TIMESTAMP,
+  pull_time     timestamp,
+  extra_attrs   text,
+  annotations   jsonb,
+  CONSTRAINT unique_artifact UNIQUE (repository_id, digest)
+*/
+
+/*
+Add new column artifact_type for artifact table to work with oci-spec v1.1.0 list referrer api
+*/
+ALTER TABLE artifact ADD COLUMN artifact_type varchar(255) NOT NULL Default '';
+
+/*set the artifact_type
+UPDATE artifact
+    SET artifact_type = media_type
+    WHERE media_type = 'application/vnd.oci.image.config.v1+json';
+
+
+ALTER TABLE artifact ALTER COLUMN media_type SET NOT NULL;
+*/
diff --git a/src/controller/artifact/abstractor.go b/src/controller/artifact/abstractor.go
@@ -127,10 +127,18 @@ func (a *abstractor) abstractManifestV2Metadata(artifact *artifact.Artifact, con
 	}
 	// use the "manifest.config.mediatype" as the media type of the artifact
 	artifact.MediaType = manifest.Config.MediaType
-
 	if manifest.Annotations[wasm.AnnotationVariantKey] == wasm.AnnotationVariantValue || manifest.Annotations[wasm.AnnotationHandlerKey] == wasm.AnnotationHandlerValue {
 		artifact.MediaType = wasm.MediaType
 	}
+	/*
+		https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers
+		For referrers list, if the artifactType is empty or missing in the image manifest, the value of artifactType MUST be set to the config descriptor mediaType value
+	*/
+	if manifest.ArtifactType != "" {
+		artifact.ArtifactType = manifest.ArtifactType
+	} else {
+		artifact.ArtifactType = manifest.Config.MediaType
+	}
 
 	// set size
 	artifact.Size = int64(len(content)) + manifest.Config.Size
@@ -153,6 +161,16 @@ func (a *abstractor) abstractIndexMetadata(ctx context.Context, art *artifact.Ar
 		return err
 	}
 
+	/*
+		https://github.com/opencontainers/distribution-spec/blob/v1.1.0/spec.md#listing-referrers
+		For referrers list, If the artifactType is empty or missing in an index, the artifactType MUST be omitted.
+	*/
+	if index.ArtifactType != "" {
+		art.ArtifactType = index.ArtifactType
+	} else {
+		art.ArtifactType = ""
+	}
+
 	// set annotations
 	art.Annotations = index.Annotations
 

diff --git a/src/controller/artifact/abstractor_test.go b/src/controller/artifact/abstractor_test.go
@@ -15,6 +15,7 @@
 package artifact
 
 import (
+	"context"
 	"testing"
 
 	"github.com/docker/distribution"
@@ -175,7 +176,66 @@ var (
     "com.example.key1": "value1"
   }
 }`
-
+	OCIManifest = `{ 
+   "schemaVersion": 2,
+   "mediaType": "application/vnd.oci.image.manifest.v1+json",
+   "config": {
+      "mediaType": "application/vnd.example.config.v1+json",
+      "digest": "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03",
+      "size": 123
+   },
+   "layers": [
+      {
+         "mediaType": "application/vnd.example.data.v1.tar+gzip",
+         "digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
+         "size": 1234
+      }
+   ],
+   "annotations": {
+      "com.example.key1": "value1"
+   }
+}`
+	OCIManifestWithArtifactType = `{
+   "schemaVersion": 2,
+   "mediaType": "application/vnd.oci.image.manifest.v1+json",
+   "artifactType": "application/vnd.example+type",
+   "config": {
+      "mediaType": "application/vnd.example.config.v1+json",
+      "digest": "sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03",
+      "size": 123
+   },
+   "layers": [
+      {
+         "mediaType": "application/vnd.example.data.v1.tar+gzip",
+         "digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
+         "size": 1234
+      }
+   ],
+   "annotations": {
+      "com.example.key1": "value1"
+   }
+}`
+	OCIManifestWithEmptyConfig = `{
+   "schemaVersion": 2,
+   "mediaType": "application/vnd.oci.image.manifest.v1+json",
+   "artifactType": "application/vnd.example+type",
+   "config": {
+     "mediaType": "application/vnd.oci.empty.v1+json",
+     "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a",
+     "size": 2
+   },
+   "layers": [
+     {
+       "mediaType": "application/vnd.example+type",
+       "digest": "sha256:e258d248fda94c63753607f7c4494ee0fcbe92f1a76bfdac795c9d84101eb317",
+       "size": 1234
+     }
+   ],
+   "annotations": {
+     "oci.opencontainers.image.created": "2023-01-02T03:04:05Z",
+     "com.example.data": "payload"
+   }
+}`
 	index = `{
   "schemaVersion": 2,
   "manifests": [
@@ -202,6 +262,34 @@ var (
     "com.example.key1": "value1"
   }
 }`
+	indexWithArtifactType = `{
+   "schemaVersion": 2,
+   "mediaType": "application/vnd.oci.image.index.v1+json",
+   "artifactType": "application/vnd.food.stand",
+   "manifests": [
+     {
+       "mediaType": "application/vnd.oci.image.manifest.v1+json",
+       "size": 7143,
+       "digest": "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
+       "platform": {
+         "architecture": "ppc64le",
+         "os": "linux"
+       }
+     },
+     {
+       "mediaType": "application/vnd.oci.image.manifest.v1+json",
+       "size": 7682,
+       "digest": "sha256:5b0bcabd1ed22e9fb1310cf6c2dec7cdef19f0ad69efa1f392e94a4333501270",
+       "platform": {
+         "architecture": "amd64",
+         "os": "linux"
+       }
+     }
+   ],
+   "annotations": {
+     "com.example.key1": "value1"
+   }
+ }`
 )
 
 type abstractorTestSuite struct {
@@ -267,6 +355,67 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfV2Manifest() {
 	a.Equal("value1", artifact.Annotations["com.example.key1"])
 }
 
+// oci-spec v1
+func (a *abstractorTestSuite) TestAbstractMetadataOfOCIManifest() {
+	manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifest))
+	a.Require().Nil(err)
+	a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil)
+	artifact := &artifact.Artifact{
+		ID: 1,
+	}
+	a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
+	a.Require().Nil(err)
+	a.Assert().Equal(int64(1), artifact.ID)
+	a.Assert().Equal(v1.MediaTypeImageManifest, artifact.ManifestMediaType)
+	a.Assert().Equal("application/vnd.example.config.v1+json", artifact.MediaType)
+	a.Assert().Equal("application/vnd.example.config.v1+json", artifact.ArtifactType)
+	a.Assert().Equal(int64(1916), artifact.Size)
+	a.Require().Len(artifact.Annotations, 1)
+	a.Equal("value1", artifact.Annotations["com.example.key1"])
+}
+
+// oci-spec v1.1.0 with artifactType
+func (a *abstractorTestSuite) TestAbstractMetadataOfOCIManifestWithArtifactType() {
+	manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithArtifactType))
+	a.Require().Nil(err)
+	a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil)
+	artifact := &artifact.Artifact{
+		ID: 1,
+	}
+	a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
+	a.Require().Nil(err)
+	a.Assert().Equal(int64(1), artifact.ID)
+	a.Assert().Equal(v1.MediaTypeImageManifest, artifact.ManifestMediaType)
+	a.Assert().Equal("application/vnd.example.config.v1+json", artifact.MediaType)
+	a.Assert().Equal("application/vnd.example+type", artifact.ArtifactType)
+	a.Assert().Equal(int64(1966), artifact.Size)
+	a.Require().Len(artifact.Annotations, 1)
+	a.Equal("value1", artifact.Annotations["com.example.key1"])
+}
+
+// empty config with artifactType
+func (a *abstractorTestSuite) TestAbstractMetadataOfV2ManifestWithEmptyConfig() {
+	// v1.MediaTypeImageManifest
+	manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageManifest, []byte(OCIManifestWithEmptyConfig))
+	a.Require().Nil(err)
+	a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil)
+	artifact := &artifact.Artifact{
+		ID: 1,
+	}
+	a.processor.On("AbstractMetadata", mock.Anything, mock.Anything, mock.Anything).Return(nil)
+	err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
+	a.Require().Nil(err)
+	a.Assert().Equal(int64(1), artifact.ID)
+	a.Assert().Equal(v1.MediaTypeImageManifest, artifact.ManifestMediaType)
+	a.Assert().Equal(v1.MediaTypeEmptyJSON, artifact.MediaType)
+	a.Assert().Equal("application/vnd.example+type", artifact.ArtifactType)
+	a.Assert().Equal(int64(1880), artifact.Size)
+	a.Require().Len(artifact.Annotations, 2)
+	a.Equal("payload", artifact.Annotations["com.example.data"])
+}
+
 // OCI index
 func (a *abstractorTestSuite) TestAbstractMetadataOfIndex() {
 	manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageIndex, []byte(index))
@@ -279,17 +428,41 @@ func (a *abstractorTestSuite) TestAbstractMetadataOfIndex() {
 	artifact := &artifact.Artifact{
 		ID: 1,
 	}
-	err = a.abstractor.AbstractMetadata(nil, artifact)
+	err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
 	a.Require().Nil(err)
 	a.Assert().Equal(int64(1), artifact.ID)
 	a.Assert().Equal(v1.MediaTypeImageIndex, artifact.ManifestMediaType)
 	a.Assert().Equal(v1.MediaTypeImageIndex, artifact.MediaType)
+	a.Assert().Equal("", artifact.ArtifactType)
 	a.Assert().Equal(int64(668), artifact.Size)
 	a.Require().Len(artifact.Annotations, 1)
 	a.Assert().Equal("value1", artifact.Annotations["com.example.key1"])
 	a.Len(artifact.References, 2)
 }
 
+func (a *abstractorTestSuite) TestAbstractMetadataOfIndexWithArtifactType() {
+	manifest, _, err := distribution.UnmarshalManifest(v1.MediaTypeImageIndex, []byte(indexWithArtifactType))
+	a.Require().Nil(err)
+	a.regCli.On("PullManifest", mock.Anything, mock.Anything).Return(manifest, "", nil)
+	a.argMgr.On("GetByDigest", mock.Anything, mock.Anything, mock.Anything).Return(&artifact.Artifact{
+		ID:   2,
+		Size: 10,
+	}, nil)
+	artifact := &artifact.Artifact{
+		ID: 1,
+	}
+	err = a.abstractor.AbstractMetadata(context.TODO(), artifact)
+	a.Require().Nil(err)
+	a.Assert().Equal(int64(1), artifact.ID)
+	a.Assert().Equal(v1.MediaTypeImageIndex, artifact.ManifestMediaType)
+	a.Assert().Equal(v1.MediaTypeImageIndex, artifact.MediaType)
+	a.Assert().Equal("application/vnd.food.stand", artifact.ArtifactType)
+	a.Assert().Equal(int64(801), artifact.Size)
+	a.Require().Len(artifact.Annotations, 1)
+	a.Assert().Equal("value1", artifact.Annotations["com.example.key1"])
+	a.Len(artifact.References, 2)
+}
+
 type unknownManifest struct{}
 
 func (u *unknownManifest) References() []distribution.Descriptor {

diff --git a/src/controller/artifact/processor/default.go b/src/controller/artifact/processor/default.go
@@ -17,6 +17,7 @@ package processor
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"regexp"
 	"strings"
 
@@ -101,6 +102,7 @@ func (d *defaultProcessor) AbstractMetadata(ctx context.Context, artifact *artif
 	if err := json.Unmarshal(manifest, mani); err != nil {
 		return err
 	}
+	fmt.Println(artifact.MediaType)
 	// if artifact.MediaType match regex, will set artifact.ExtraAttrs
 	if d.GetArtifactType(ctx, artifact) != ArtifactTypeUnknown {
 		// get config layer