(feature) Implement polling image source in intervals

api/core/v1alpha1/catalog_types.go

@@ -70,6 +70,7 @@ type CatalogList struct {
 }
 
 // CatalogSpec defines the desired state of Catalog
+// +kubebuilder:validation:XValidation:rule="!has(self.source.image.pollInterval) || (self.source.image.pollInterval == \"\") || (self.source.image.ref.find('@sha256:') == \"\")",message="cannot specify PollInterval while using digest-based image"
 type CatalogSpec struct {
 	// Source is the source of a Catalog that contains Operators' metadata in the FBC format
 	// https://olm.operatorframework.io/docs/reference/file-based-catalogs/#docs
@@ -81,11 +82,16 @@ type CatalogStatus struct {
 	// Conditions store the status conditions of the Catalog instances
 	Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,1,rep,name=conditions"`
 
-	ResolvedSource *CatalogSource `json:"resolvedSource,omitempty"`
-	Phase          string         `json:"phase,omitempty"`
+	// ResolvedSource contains information about the resolved source
+	ResolvedSource *ResolvedCatalogSource `json:"resolvedSource,omitempty"`
+	Phase          string                 `json:"phase,omitempty"`
 	// ContentURL is a cluster-internal address that on-cluster components
 	// can read the content of a catalog from
 	ContentURL string `json:"contentURL,omitempty"`
+	// observedGeneration is the most recent generation observed for this Catalog. It corresponds to the
+	// Catalog's generation, which is updated on mutation by the API Server.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 }
 
 // CatalogSource contains the sourcing information for a Catalog
@@ -96,12 +102,36 @@ type CatalogSource struct {
 	Image *ImageSource `json:"image,omitempty"`
 }
 
+// ResolvedCatalogSource contains the information about a sourced Catalog
+type ResolvedCatalogSource struct {
+	// Type defines the kind of Catalog content that was sourced.
+	Type SourceType `json:"type"`
+	// Image is the catalog image that backs the content of this catalog.
+	Image *ResolvedImageSource `json:"image,omitempty"`
+}
+
+// ResolvedImageSource contains information about the sourced Catalog
+type ResolvedImageSource struct {
+	// Ref contains the reference to a container image containing Catalog contents.
+	Ref string `json:"ref"`
+	// ResolvedRef contains the resolved sha256 image ref containing Catalog contents.
+	ResolvedRef string `json:"resolvedRef"`
+	// LastPollAtempt is the time when the source resolved was last polled for new content.
+	LastPollAttempt metav1.Time `json:"lastPollAttempt"`
+	// pullSecret exists to retain compatibility with the existing v1alpha1 APIs. It will be removed in v1alpha2.
+	PullSecret string `json:"pullSecret,omitempty"`
+}
+
 // ImageSource contains information required for sourcing a Catalog from an OCI image
 type ImageSource struct {
 	// Ref contains the reference to a container image containing Catalog contents.
 	Ref string `json:"ref"`
 	// PullSecret contains the name of the image pull secret in the namespace that catalogd is deployed.
 	PullSecret string `json:"pullSecret,omitempty"`
+	// PollInterval indicates the interval at which the image source should be polled for new content,
+	// specified as a duration (e.g., "5m", "1h", "24h", "etc".). Note that PollInterval may not be
+	// specified for a catalog image referenced by a sha256 digest.
+	PollInterval *metav1.Duration `json:"pollInterval,omitempty"`
 }
 
 func init() {

api/core/v1alpha1/catalog_types_test.go

@@ -0,0 +1,142 @@
+package v1alpha1
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apiextensions-apiserver/pkg/apiserver/schema"
+	"k8s.io/apiextensions-apiserver/pkg/apiserver/schema/cel"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	celconfig "k8s.io/apiserver/pkg/apis/cel"
+	"sigs.k8s.io/yaml"
+)
+
+func TestPollIntervalCELValidationRules(t *testing.T) {
+	validators := fieldValidatorsFromFile(t, "../../../config/crd/bases/catalogd.operatorframework.io_catalogs.yaml")
+	pth := "openAPIV3Schema.properties.spec"
+	validator, found := validators["v1alpha1"][pth]
+	assert.True(t, found)
+
+	for name, tc := range map[string]struct {
+		spec     CatalogSpec
+		wantErrs []string
+	}{
+		"digest based image ref, poll interval not allowed, poll interval specified": {
+			spec: CatalogSpec{
+				Source: CatalogSource{
+					Type: SourceTypeImage,
+					Image: &ImageSource{
+						Ref:          "docker.io/test-image@sha256:asdf98234sd",
+						PollInterval: &metav1.Duration{Duration: time.Minute},
+					},
+				},
+			},
+			wantErrs: []string{
+				"openAPIV3Schema.properties.spec: Invalid value: \"object\": cannot specify PollInterval while using digest-based image",
+			},
+		},
+		"digest based image ref, poll interval not allowed, poll interval not specified": {
+			spec: CatalogSpec{
+				Source: CatalogSource{
+					Type: SourceTypeImage,
+					Image: &ImageSource{
+						Ref: "docker.io/example/test-catalog@sha256:asdf123",
+					},
+				},
+			},
+			wantErrs: []string{},
+		},
+	} {
+		t.Run(name, func(t *testing.T) {
+			obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&tc.spec)
+			require.NoError(t, err)
+			errs := validator(obj, nil)
+			require.Equal(t, len(tc.wantErrs), len(errs))
+			for i := range tc.wantErrs {
+				got := errs[i].Error()
+				assert.Equal(t, tc.wantErrs[i], got)
+			}
+		})
+	}
+}
+
+// fieldValidatorsFromFile extracts the CEL validators by version and JSONPath from a CRD file and returns
+// a validator func for testing against samples.
+func fieldValidatorsFromFile(t *testing.T, crdFilePath string) map[string]map[string]CELValidateFunc {
+	data, err := os.ReadFile(crdFilePath)
+	require.NoError(t, err)
+
+	var crd apiextensionsv1.CustomResourceDefinition
+	err = yaml.Unmarshal(data, &crd)
+	require.NoError(t, err)
+
+	ret := map[string]map[string]CELValidateFunc{}
+	for _, v := range crd.Spec.Versions {
+		var internalSchema apiextensions.JSONSchemaProps
+		err := apiextensionsv1.Convert_v1_JSONSchemaProps_To_apiextensions_JSONSchemaProps(v.Schema.OpenAPIV3Schema, &internalSchema, nil)
+		require.NoError(t, err, "failed to convert JSONSchemaProps for version %s: %v", v.Name, err)
+		structuralSchema, err := schema.NewStructural(&internalSchema)
+		require.NoError(t, err, "failed to create StructuralSchema for version %s: %v", v.Name, err)
+
+		versionVals, err := findCEL(structuralSchema, true, field.NewPath("openAPIV3Schema"))
+		require.NoError(t, err, "failed to find CEL for version %s: %v", v.Name, err)
+		ret[v.Name] = versionVals
+	}
+	return ret
+}
+
+// CELValidateFunc tests a sample object against a CEL validator.
+type CELValidateFunc func(obj, old interface{}) field.ErrorList
+
+func findCEL(s *schema.Structural, root bool, pth *field.Path) (map[string]CELValidateFunc, error) {
+	ret := map[string]CELValidateFunc{}
+
+	if len(s.XValidations) > 0 {
+		s := *s
+		pth := *pth
+		ret[pth.String()] = func(obj, old interface{}) field.ErrorList {
+			errs, _ := cel.NewValidator(&s, root, celconfig.PerCallLimit).Validate(context.TODO(), &pth, &s, obj, old, celconfig.RuntimeCELCostBudget)
+			return errs
+		}
+	}
+
+	for k, v := range s.Properties {
+		v := v
+		sub, err := findCEL(&v, false, pth.Child("properties").Child(k))
+		if err != nil {
+			return nil, err
+		}
+
+		for pth, val := range sub {
+			ret[pth] = val
+		}
+	}
+	if s.Items != nil {
+		sub, err := findCEL(s.Items, false, pth.Child("items"))
+		if err != nil {
+			return nil, err
+		}
+		for pth, val := range sub {
+			ret[pth] = val
+		}
+	}
+	if s.AdditionalProperties != nil && s.AdditionalProperties.Structural != nil {
+		sub, err := findCEL(s.AdditionalProperties.Structural, false, pth.Child("additionalProperties"))
+		if err != nil {
+			return nil, err
+		}
+		for pth, val := range sub {
+			ret[pth] = val
+		}
+	}
+
+	return ret, nil
+}

config/crd/bases/catalogd.operatorframework.io_catalogs.yaml

@@ -49,6 +49,13 @@ spec:
                     description: Image is the catalog image that backs the content
                       of this catalog.
                     properties:
+                      pollInterval:
+                        description: PollInterval indicates the interval at which
+                          the image source should be polled for new content, specified
+                          as a duration (e.g., "5m", "1h", "24h", "etc".). Note that
+                          PollInterval may not be specified for a catalog image referenced
+                          by a sha256 digest.
+                        type: string
                       pullSecret:
                         description: PullSecret contains the name of the image pull
                           secret in the namespace that catalogd is deployed.
@@ -69,6 +76,10 @@ spec:
             required:
             - source
             type: object
+            x-kubernetes-validations:
+            - message: cannot specify PollInterval while using digest-based image
+              rule: '!has(self.source.image.pollInterval) || (self.source.image.pollInterval
+                == "") || (self.source.image.ref.find(''@sha256:'') == "")'
           status:
             description: CatalogStatus defines the observed state of Catalog
             properties:
@@ -146,29 +157,47 @@ spec:
                 description: ContentURL is a cluster-internal address that on-cluster
                   components can read the content of a catalog from
                 type: string
+              observedGeneration:
+                description: observedGeneration is the most recent generation observed
+                  for this Catalog. It corresponds to the Catalog's generation, which
+                  is updated on mutation by the API Server.
+                format: int64
+                type: integer
               phase:
                 type: string
               resolvedSource:
-                description: CatalogSource contains the sourcing information for a
-                  Catalog
+                description: ResolvedSource contains information about the resolved
+                  source
                 properties:
                   image:
                     description: Image is the catalog image that backs the content
                       of this catalog.
                     properties:
+                      lastPollAttempt:
+                        description: LastPollAtempt is the time when the source resolved
+                          was last polled for new content.
+                        format: date-time
+                        type: string
                       pullSecret:
-                        description: PullSecret contains the name of the image pull
-                          secret in the namespace that catalogd is deployed.
+                        description: pullSecret exists to retain compatibility with
+                          the existing v1alpha1 APIs. It will be removed in v1alpha2.
                         type: string
                       ref:
                         description: Ref contains the reference to a container image
                           containing Catalog contents.
                         type: string
+                      resolvedRef:
+                        description: ResolvedRef contains the resolved sha256 image
+                          ref containing Catalog contents.
+                        type: string
                     required:
+                    - lastPollAttempt
                     - ref
+                    - resolvedRef
                     type: object
                   type:
-                    description: Type defines the kind of Catalog content being sourced.
+                    description: Type defines the kind of Catalog content that was
+                      sourced.
                     type: string
                 required:
                 - type

config/samples/core_v1alpha1_catalog.yaml

@@ -6,4 +6,5 @@ spec:
   source:
     type: image
     image:
+      pollInterval: 24h
       ref: quay.io/operatorhubio/catalog:latest