(feature) Implement polling image source in intervals

Implements https://docs.google.com/document/d/1iWSrWL9pYRJ5Ua3VYErkK1Q2lAusBUeDCh66Ew4lDbQ/edit?usp=sharing

Closes #180

Signed-off-by: Anik Bhattacharjee <anbhatta@redhat.com>

api/core/v1alpha1/catalog_types.go

@@ -70,6 +70,7 @@ type CatalogList struct {
 }
 
 // CatalogSpec defines the desired state of Catalog
+// +kubebuilder:validation:XValidation:rule="!has(self.source.image.pollInterval) || (self.source.image.ref.find('sha256') != \"\" && self.source.image.pollInterval == \"\") || (self.source.image.ref.find('sha256') == \"\" && self.source.image.pollInterval == \"\") || (self.source.image.ref.find('sha256') == \"\" && self.source.image.pollInterval != \"\") ",message="cannot specify PollInterval while using tagged image"
 type CatalogSpec struct {
 	// Source is the source of a Catalog that contains Operators' metadata in the FBC format
 	// https://olm.operatorframework.io/docs/reference/file-based-catalogs/#docs
@@ -86,6 +87,8 @@ type CatalogStatus struct {
 	// ContentURL is a cluster-internal address that on-cluster components
 	// can read the content of a catalog from
 	ContentURL string `json:"contentURL,omitempty"`
+	// LastPollAttempt logs the time when the image source was polled for new content.
+	LastPollAttempt metav1.Time `json:"lastPollAttempt,omitempty"`
 }
 
 // CatalogSource contains the sourcing information for a Catalog
@@ -102,6 +105,12 @@ type ImageSource struct {
 	Ref string `json:"ref"`
 	// PullSecret contains the name of the image pull secret in the namespace that catalogd is deployed.
 	PullSecret string `json:"pullSecret,omitempty"`
+	// PollInterval indicates the interval at which the image source should be polled for new content,
+	// specified as a duration (e.g., "5m", "1h", "24h", "etc".).
+	// A default value of "24h" is used if not specified. Specify "0s" to disable polling. Note that
+	// PollInterval may not be specified for a catalog image referenced by a sha256 digest.
+	// +kubebuilder:default:="24h"
+	PollInterval metav1.Duration `json:"pollInterval"`
 }
 
 func init() {

config/crd/bases/catalogd.operatorframework.io_catalogs.yaml

@@ -49,6 +49,15 @@ spec:
                     description: Image is the catalog image that backs the content
                       of this catalog.
                     properties:
+                      pollInterval:
+                        default: 24h
+                        description: PollInterval indicates the interval at which
+                          the image source should be polled for new content, specified
+                          as a duration (e.g., "5m", "1h", "24h", "etc".). A default
+                          value of "24h" is used if not specified. Specify "0s" to
+                          disable polling. Note that PollInterval may not be specified
+                          for a catalog image referenced by a sha256 digest.
+                        type: string
                       pullSecret:
                         description: PullSecret contains the name of the image pull
                           secret in the namespace that catalogd is deployed.
@@ -58,6 +67,7 @@ spec:
                           containing Catalog contents.
                         type: string
                     required:
+                    - pollInterval
                     - ref
                     type: object
                   type:
@@ -69,6 +79,12 @@ spec:
             required:
             - source
             type: object
+            x-kubernetes-validations:
+            - message: cannot specify PollInterval while using tagged image
+              rule: '!has(self.source.image.pollInterval) || (self.source.image.ref.find(''sha256'')
+                != "" && self.source.image.pollInterval == "") || (self.source.image.ref.find(''sha256'')
+                == "" && self.source.image.pollInterval == "") || (self.source.image.ref.find(''sha256'')
+                == "" && self.source.image.pollInterval != "") '
           status:
             description: CatalogStatus defines the observed state of Catalog
             properties:
@@ -146,6 +162,11 @@ spec:
                 description: ContentURL is a cluster-internal address that on-cluster
                   components can read the content of a catalog from
                 type: string
+              lastPollAttempt:
+                description: LastPollAttempt logs the time when the image source was
+                  polled for new content.
+                format: date-time
+                type: string
               phase:
                 type: string
               resolvedSource:
@@ -156,6 +177,15 @@ spec:
                     description: Image is the catalog image that backs the content
                       of this catalog.
                     properties:
+                      pollInterval:
+                        default: 24h
+                        description: PollInterval indicates the interval at which
+                          the image source should be polled for new content, specified
+                          as a duration (e.g., "5m", "1h", "24h", "etc".). A default
+                          value of "24h" is used if not specified. Specify "0s" to
+                          disable polling. Note that PollInterval may not be specified
+                          for a catalog image referenced by a sha256 digest.
+                        type: string
                       pullSecret:
                         description: PullSecret contains the name of the image pull
                           secret in the namespace that catalogd is deployed.
@@ -165,6 +195,7 @@ spec:
                           containing Catalog contents.
                         type: string
                     required:
+                    - pollInterval
                     - ref
                     type: object
                   type:

config/samples/core_v1alpha1_catalog.yaml

@@ -6,4 +6,5 @@ spec:
   source:
     type: image
     image:
+      pollInterval: 24h
       ref: quay.io/operatorhubio/catalog:latest

internal/source/image_registry_client.go

@@ -15,6 +15,7 @@ import (
 	gcrkube "github.com/google/go-containerregistry/pkg/authn/kubernetes"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	catalogdv1alpha1 "github.com/operator-framework/catalogd/api/core/v1alpha1"
@@ -119,11 +120,13 @@ func unpackedResult(fsys fs.FS, catalog *catalogdv1alpha1.Catalog, ref string) *
 		ResolvedSource: &catalogdv1alpha1.CatalogSource{
 			Type: catalogdv1alpha1.SourceTypeImage,
 			Image: &catalogdv1alpha1.ImageSource{
-				Ref:        ref,
-				PullSecret: catalog.Spec.Source.Image.PullSecret,
+				Ref:          ref,
+				PullSecret:   catalog.Spec.Source.Image.PullSecret,
+				PollInterval: catalog.Spec.Source.Image.PollInterval,
 			},
 		},
-		State: StateUnpacked,
+		State:        StateUnpacked,
+		RequeueAfter: wait.Jitter(catalog.Spec.Source.Image.PollInterval.Duration, 0.5),
 	}
 }
 

internal/source/unpacker.go

@@ -6,6 +6,7 @@ import (
 	"io/fs"
 	"os"
 	"path"
+	"time"
 
 	catalogdv1alpha1 "github.com/operator-framework/catalogd/api/core/v1alpha1"
 )
@@ -57,6 +58,10 @@ type Result struct {
 	// Message is contextual information about the progress of unpacking the
 	// catalog content.
 	Message string
+
+	// RequeueAfter contains the duration after which the controller should set
+	// up for Unpacking again
+	RequeueAfter time.Duration
 }
 
 type State string

pkg/controllers/core/catalog_controller.go

@@ -152,7 +152,7 @@ func (r *CatalogReconciler) reconcile(ctx context.Context, catalog *v1alpha1.Cat
 		}
 
 		updateStatusUnpacked(&catalog.Status, unpackResult, contentURL)
-		return ctrl.Result{}, nil
+		return ctrl.Result{RequeueAfter: unpackResult.RequeueAfter}, nil
 	default:
 		return ctrl.Result{}, updateStatusUnpackFailing(&catalog.Status, fmt.Errorf("unknown unpack state %q: %v", unpackResult.State, err))
 	}
@@ -184,6 +184,7 @@ func updateStatusUnpacked(status *v1alpha1.CatalogStatus, result *source.Result,
 	status.ResolvedSource = result.ResolvedSource
 	status.ContentURL = contentURL
 	status.Phase = v1alpha1.PhaseUnpacked
+	status.LastPollAttempt = metav1.Now()
 	meta.SetStatusCondition(&status.Conditions, metav1.Condition{
 		Type:    v1alpha1.TypeUnpacked,
 		Status:  metav1.ConditionTrue,

pkg/controllers/core/catalog_controller_test.go

@@ -135,43 +135,6 @@ var _ = Describe("Catalogd Controller Test", func() {
 		BeforeEach(func() {
 			catalogKey = types.NamespacedName{Name: fmt.Sprintf("catalogd-test-%s", rand.String(8))}
 		})
-
-		When("the catalog specifies an invalid source", func() {
-			BeforeEach(func() {
-				By("initializing cluster state")
-				catalog = &v1alpha1.Catalog{
-					ObjectMeta: metav1.ObjectMeta{Name: catalogKey.Name},
-					Spec: v1alpha1.CatalogSpec{
-						Source: v1alpha1.CatalogSource{
-							Type: "invalid-source",
-						},
-					},
-				}
-				Expect(cl.Create(ctx, catalog)).To(Succeed())
-			})
-
-			AfterEach(func() {
-				By("tearing down cluster state")
-				Expect(cl.Delete(ctx, catalog)).To(Succeed())
-			})
-
-			It("should set unpacking status to failed and return an error", func() {
-				res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: catalogKey})
-				Expect(res).To(Equal(ctrl.Result{}))
-				Expect(err).To(HaveOccurred())
-
-				// get the catalog and ensure status is set properly
-				cat := &v1alpha1.Catalog{}
-				Expect(cl.Get(ctx, catalogKey, cat)).To(Succeed())
-				Expect(cat.Status.ResolvedSource).To(BeNil())
-				Expect(cat.Status.Phase).To(Equal(v1alpha1.PhaseFailing))
-				cond := meta.FindStatusCondition(cat.Status.Conditions, v1alpha1.TypeUnpacked)
-				Expect(cond).ToNot(BeNil())
-				Expect(cond.Reason).To(Equal(v1alpha1.ReasonUnpackFailed))
-				Expect(cond.Status).To(Equal(metav1.ConditionFalse))
-			})
-		})
-
 		When("the catalog specifies a valid source", func() {
 			BeforeEach(func() {
 				By("initializing cluster state")

test/e2e/cel_validation_test.go

@@ -0,0 +1,66 @@
+package e2e
+
+import (
+	"context"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	catalogd "github.com/operator-framework/catalogd/api/core/v1alpha1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+var _ = Describe("CEL validation tests", func() {
+	var (
+		ctx     context.Context
+		catalog *catalogd.Catalog
+	)
+	When("A Catalog with a digest based image ref is created with a poll interval", func() {
+		duration45m, _ := time.ParseDuration("45m")
+		BeforeEach(func() {
+			ctx = context.Background()
+
+			catalog = &catalogd.Catalog{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: catalogName,
+				},
+				Spec: catalogd.CatalogSpec{
+					Source: catalogd.CatalogSource{
+						Type: catalogd.SourceTypeImage,
+						Image: &catalogd.ImageSource{
+							Ref:          "docker.io/test-image@sha256:asdf98234sd",
+							PollInterval: metav1.Duration{Duration: duration45m},
+						},
+					},
+				},
+			}
+		})
+		It("throws an error", func() {
+			err = c.Create(ctx, catalog)
+			Expect(err).To(HaveOccurred())
+		})
+	})
+	When("A Catalog with a digest based image ref is created without a poll interval", func() {
+		BeforeEach(func() {
+			ctx = context.Background()
+
+			catalog = &catalogd.Catalog{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: catalogName,
+				},
+				Spec: catalogd.CatalogSpec{
+					Source: catalogd.CatalogSource{
+						Type: catalogd.SourceTypeImage,
+						Image: &catalogd.ImageSource{
+							Ref: "docker.io/test-image@sha256:asdf98234sd",
+						},
+					},
+				},
+			}
+		})
+		It("does not throw an error", func() {
+			err = c.Create(ctx, catalog)
+			Expect(err).ToNot(HaveOccurred())
+		})
+	})
+})