(feature) Implement polling image source in intervals

Implements https://docs.google.com/document/d/1iWSrWL9pYRJ5Ua3VYErkK1Q2lAusBUeDCh66Ew4lDbQ/edit?usp=sharing

Closes #180

Signed-off-by: Anik Bhattacharjee <anbhatta@redhat.com>

api/core/v1alpha1/catalog_types.go

@@ -70,6 +70,7 @@ type CatalogList struct {
 }
 
 // CatalogSpec defines the desired state of Catalog
+// +kubebuilder:validation:XValidation:rule="!has(self.source.image.pollInterval) || (self.source.image.ref.find('sha256') != \"\" && self.source.image.pollInterval == \"\") || (self.source.image.ref.find('sha256') == \"\" && self.source.image.pollInterval == \"\") || (self.source.image.ref.find('sha256') == \"\" && self.source.image.pollInterval != \"\") ",message="cannot specify PollInterval while using tagged image"
 type CatalogSpec struct {
 	// Source is the source of a Catalog that contains Operators' metadata in the FBC format
 	// https://olm.operatorframework.io/docs/reference/file-based-catalogs/#docs
@@ -86,6 +87,8 @@ type CatalogStatus struct {
 	// ContentURL is a cluster-internal address that on-cluster components
 	// can read the content of a catalog from
 	ContentURL string `json:"contentURL,omitempty"`
+	// LastPollAttempt logs the time when the image source was polled for new content.
+	LastPollAttempt metav1.Time `json:"lastPollAttempt,omitempty"`
 }
 
 // CatalogSource contains the sourcing information for a Catalog
@@ -102,6 +105,12 @@ type ImageSource struct {
 	Ref string `json:"ref"`
 	// PullSecret contains the name of the image pull secret in the namespace that catalogd is deployed.
 	PullSecret string `json:"pullSecret,omitempty"`
+	// PollInterval indicates the interval at which the image source should be polled for new content,
+	// specified as a duration (e.g., 5m, 1h, 24h, etc.).
+	// A default value of 24h is used if not specified. Specify 0 to disable polling. Note that
+	// PollInterval may not be specified for a catalog image referenced by a sha256 digest.
+	// +kubebuilder:default:="24h"
+	PollInterval metav1.Duration `json:"pollInterval"`
 }
 
 func init() {

config/crd/bases/catalogd.operatorframework.io_catalogs.yaml

@@ -49,6 +49,15 @@ spec:
                     description: Image is the catalog image that backs the content
                       of this catalog.
                     properties:
+                      pollInterval:
+                        default: 24h
+                        description: PollInterval indicates the interval at which
+                          the image source should be polled for new content, specified
+                          as a duration (e.g., 5m, 1h, 24h, etc.). A default value
+                          of 24h is used if not specified. Specify 0 to disable polling.
+                          Note that PollInterval may not be specified for a catalog
+                          image referenced by a sha256 digest.
+                        type: string
                       pullSecret:
                         description: PullSecret contains the name of the image pull
                           secret in the namespace that catalogd is deployed.
@@ -58,6 +67,7 @@ spec:
                           containing Catalog contents.
                         type: string
                     required:
+                    - pollInterval
                     - ref
                     type: object
                   type:
@@ -69,6 +79,12 @@ spec:
             required:
             - source
             type: object
+            x-kubernetes-validations:
+            - message: cannot specify PollInterval while using tagged image
+              rule: '!has(self.source.image.pollInterval) || (self.source.image.ref.find(''sha256'')
+                != "" && self.source.image.pollInterval == "") || (self.source.image.ref.find(''sha256'')
+                == "" && self.source.image.pollInterval == "") || (self.source.image.ref.find(''sha256'')
+                == "" && self.source.image.pollInterval != "") '
           status:
             description: CatalogStatus defines the observed state of Catalog
             properties:
@@ -146,6 +162,11 @@ spec:
                 description: ContentURL is a cluster-internal address that on-cluster
                   components can read the content of a catalog from
                 type: string
+              lastPollAttempt:
+                description: LastPollAttempt logs the time when the image source was
+                  polled for new content.
+                format: date-time
+                type: string
               phase:
                 type: string
               resolvedSource:
@@ -156,6 +177,15 @@ spec:
                     description: Image is the catalog image that backs the content
                       of this catalog.
                     properties:
+                      pollInterval:
+                        default: 24h
+                        description: PollInterval indicates the interval at which
+                          the image source should be polled for new content, specified
+                          as a duration (e.g., 5m, 1h, 24h, etc.). A default value
+                          of 24h is used if not specified. Specify 0 to disable polling.
+                          Note that PollInterval may not be specified for a catalog
+                          image referenced by a sha256 digest.
+                        type: string
                       pullSecret:
                         description: PullSecret contains the name of the image pull
                           secret in the namespace that catalogd is deployed.
@@ -165,6 +195,7 @@ spec:
                           containing Catalog contents.
                         type: string
                     required:
+                    - pollInterval
                     - ref
                     type: object
                   type:

config/samples/core_v1alpha1_catalog.yaml

@@ -6,4 +6,5 @@ spec:
   source:
     type: image
     image:
+      pollInterval: 24h
       ref: quay.io/operatorhubio/catalog:latest

internal/source/image_registry_client.go

@@ -15,6 +15,8 @@ import (
 	gcrkube "github.com/google/go-containerregistry/pkg/authn/kubernetes"
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/v1/remote"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	catalogdv1alpha1 "github.com/operator-framework/catalogd/api/core/v1alpha1"
@@ -106,7 +108,7 @@ func (i *ImageRegistry) Cleanup(_ context.Context, catalog *catalogdv1alpha1.Cat
 }
 
 func unpackedResult(fsys fs.FS, catalog *catalogdv1alpha1.Catalog, ref string) *Result {
-	return &Result{
+	result := &Result{
 		FS: fsys,
 		ResolvedSource: &catalogdv1alpha1.CatalogSource{
 			Type: catalogdv1alpha1.SourceTypeImage,
@@ -117,6 +119,12 @@ func unpackedResult(fsys fs.FS, catalog *catalogdv1alpha1.Catalog, ref string) *
 		},
 		State: StateUnpacked,
 	}
+	if catalog.Spec.Source.Image.PollInterval.Duration == 0 {
+		result.RequeueAfter = nil
+	} else {
+		result.RequeueAfter = &metav1.Duration{Duration: wait.Jitter(catalog.Spec.Source.Image.PollInterval.Duration, 0.5)}
+	}
+	return result
 }
 
 // unpackImage unpacks a catalog image reference to the provided unpackPath,

internal/source/image_registry_client_test.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/google/go-containerregistry/pkg/name"
 	"github.com/google/go-containerregistry/pkg/registry"
@@ -361,3 +362,86 @@ func TestImageRegistry(t *testing.T) {
 		}
 	})
 }
+
+func TestPolling(t *testing.T) {
+	ctx := context.Background()
+	testCache := t.TempDir()
+	imgReg := &source.ImageRegistry{
+		BaseCachePath: testCache,
+	}
+	// Start a new server running an image registry
+	srv := httptest.NewServer(registry.New())
+	defer srv.Close()
+
+	// parse the server url so we can grab just the host
+	url, err := url.Parse(srv.URL)
+	assert.NoError(t, err)
+	duration0s, _ := time.ParseDuration("0s")
+	duration24h, _ := time.ParseDuration("24h")
+	for _, tt := range []struct {
+		testName                           string
+		inputCatalog                       v1alpha1.Catalog
+		expectedResolvedSourcePollInterval *metav1.Duration
+		shouldError                        bool
+	}{
+		{
+			testName: "When a poll interval is not specified, a default value of 24h is set in the resolved source as poll interval",
+			inputCatalog: v1alpha1.Catalog{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test",
+				},
+				Spec: v1alpha1.CatalogSpec{
+					Source: v1alpha1.CatalogSource{
+						Type: v1alpha1.SourceTypeImage,
+						Image: &v1alpha1.ImageSource{
+							Ref: fmt.Sprintf("%s/%s", url.Host, "test-image:latest"),
+						},
+					},
+				},
+			},
+			expectedResolvedSourcePollInterval: &metav1.Duration{Duration: duration24h},
+			shouldError:                        false,
+		},
+		{
+			testName: "When a poll interval of 0s is specified, no poll interval is set in the resolved source",
+			inputCatalog: v1alpha1.Catalog{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "test",
+				},
+				Spec: v1alpha1.CatalogSpec{
+					Source: v1alpha1.CatalogSource{
+						Type: v1alpha1.SourceTypeImage,
+						Image: &v1alpha1.ImageSource{
+							Ref:          fmt.Sprintf("%s/%s", url.Host, "test-image:latest"),
+							PollInterval: metav1.Duration{Duration: duration0s},
+						},
+					},
+				},
+			},
+			expectedResolvedSourcePollInterval: nil,
+			shouldError:                        false,
+		},
+	} {
+		t.Run(t.Name(), func(t *testing.T) {
+			img, err := random.Image(20, 3)
+			assert.NoError(t, err)
+			img, err = mutate.Config(img, v1.Config{
+				Labels: map[string]string{
+					source.ConfigDirLabel: "/configs",
+				},
+			})
+			assert.NoError(t, err)
+			ref, err := name.ParseReference(tt.inputCatalog.Spec.Source.Image.Ref)
+
+			assert.NoError(t, err)
+			remote.Write(ref, img)
+			res, err := imgReg.Unpack(ctx, &tt.inputCatalog)
+			if tt.shouldError {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+			}
+			assert.Equal(t, res.ResolvedSource.Image.PollInterval, tt.expectedResolvedSourcePollInterval)
+		})
+	}
+}

internal/source/unpacker.go

@@ -8,6 +8,8 @@ import (
 	"path"
 
 	catalogdv1alpha1 "github.com/operator-framework/catalogd/api/core/v1alpha1"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // TODO: This package is almost entirely copy/pasted from rukpak. We should look
@@ -57,6 +59,10 @@ type Result struct {
 	// Message is contextual information about the progress of unpacking the
 	// catalog content.
 	Message string
+
+	// RequeueAfter contains the duration after which the controller should set
+	// up for Unpacking again
+	RequeueAfter *metav1.Duration
 }
 
 type State string

pkg/controllers/core/catalog_controller.go

@@ -151,7 +151,10 @@ func (r *CatalogReconciler) reconcile(ctx context.Context, catalog *v1alpha1.Cat
 		}
 
 		updateStatusUnpacked(&catalog.Status, unpackResult, contentURL)
-		return ctrl.Result{}, nil
+		if unpackResult.RequeueAfter == nil {
+			return ctrl.Result{}, nil
+		}
+		return ctrl.Result{RequeueAfter: unpackResult.RequeueAfter.Duration}, nil
 	default:
 		return ctrl.Result{}, updateStatusUnpackFailing(&catalog.Status, fmt.Errorf("unknown unpack state %q: %v", unpackResult.State, err))
 	}
@@ -183,6 +186,7 @@ func updateStatusUnpacked(status *v1alpha1.CatalogStatus, result *source.Result,
 	status.ResolvedSource = result.ResolvedSource
 	status.ContentURL = contentURL
 	status.Phase = v1alpha1.PhaseUnpacked
+	status.LastPollAttempt = metav1.Now()
 	meta.SetStatusCondition(&status.Conditions, metav1.Condition{
 		Type:    v1alpha1.TypeUnpacked,
 		Status:  metav1.ConditionTrue,

pkg/controllers/core/catalog_controller_test.go

@@ -135,43 +135,6 @@ var _ = Describe("Catalogd Controller Test", func() {
 		BeforeEach(func() {
 			catalogKey = types.NamespacedName{Name: fmt.Sprintf("catalogd-test-%s", rand.String(8))}
 		})
-
-		When("the catalog specifies an invalid source", func() {
-			BeforeEach(func() {
-				By("initializing cluster state")
-				catalog = &v1alpha1.Catalog{
-					ObjectMeta: metav1.ObjectMeta{Name: catalogKey.Name},
-					Spec: v1alpha1.CatalogSpec{
-						Source: v1alpha1.CatalogSource{
-							Type: "invalid-source",
-						},
-					},
-				}
-				Expect(cl.Create(ctx, catalog)).To(Succeed())
-			})
-
-			AfterEach(func() {
-				By("tearing down cluster state")
-				Expect(cl.Delete(ctx, catalog)).To(Succeed())
-			})
-
-			It("should set unpacking status to failed and return an error", func() {
-				res, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: catalogKey})
-				Expect(res).To(Equal(ctrl.Result{}))
-				Expect(err).To(HaveOccurred())
-
-				// get the catalog and ensure status is set properly
-				cat := &v1alpha1.Catalog{}
-				Expect(cl.Get(ctx, catalogKey, cat)).To(Succeed())
-				Expect(cat.Status.ResolvedSource).To(BeNil())
-				Expect(cat.Status.Phase).To(Equal(v1alpha1.PhaseFailing))
-				cond := meta.FindStatusCondition(cat.Status.Conditions, v1alpha1.TypeUnpacked)
-				Expect(cond).ToNot(BeNil())
-				Expect(cond.Reason).To(Equal(v1alpha1.ReasonUnpackFailed))
-				Expect(cond.Status).To(Equal(metav1.ConditionFalse))
-			})
-		})
-
 		When("the catalog specifies a valid source", func() {
 			BeforeEach(func() {
 				By("initializing cluster state")