Upgrade addon framework to support image override (#70)

Signed-off-by: zhujian <jiazhu@redhat.com>

.github/workflows/go-presubmit.yml

@@ -87,7 +87,7 @@ jobs:
       - name: Build image
         run: |
           make images
-          kind load docker-image quay.io/open-cluster-management/managed-serviceaccount:latest  --name chart-testing
+          kind load docker-image quay.io/open-cluster-management/managed-serviceaccount:latest --name chart-testing
       - name: Install latest managed-serviceaccount
         run: |
           helm install \
@@ -125,7 +125,7 @@ jobs:
       - name: Build image
         run: |
           make images
-          kind load docker-image quay.io/open-cluster-management/managed-serviceaccount:latest  --name chart-testing
+          kind load docker-image quay.io/open-cluster-management/managed-serviceaccount:latest --name chart-testing
       - name: Install latest managed-serviceaccount
         run: |
           kubectl create namespace loopback --dry-run=client -o yaml | kubectl apply -f -

cmd/manager/main.go

@@ -40,6 +40,7 @@ import (
 	"open-cluster-management.io/addon-framework/pkg/addonfactory"
 	"open-cluster-management.io/addon-framework/pkg/addonmanager"
 	"open-cluster-management.io/addon-framework/pkg/agent"
+	"open-cluster-management.io/addon-framework/pkg/utils"
 	addonclient "open-cluster-management.io/api/client/addon/clientset/versioned"
 	authv1alpha1 "open-cluster-management.io/managed-serviceaccount/api/v1alpha1"
 	"open-cluster-management.io/managed-serviceaccount/pkg/addon/commoncontroller"
@@ -176,15 +177,21 @@ func main() {
 	}
 
 	agentFactory := addonfactory.NewAgentAddonFactory(common.AddonName, manager.FS, "manifests/templates").
-		WithConfigGVRs(addonfactory.AddOnDeploymentConfigGVR).
+		WithConfigGVRs(utils.AddOnDeploymentConfigGVR).
 		WithGetValuesFuncs(
 			manager.GetDefaultValues(addonAgentImageName, imagePullSecret),
+			addonfactory.GetAgentImageValues(
+				addonfactory.NewAddOnDeploymentConfigGetter(addonClient),
+				"Image",
+				addonAgentImageName,
+			),
 			addonfactory.GetAddOnDeloymentConfigValues(
-				addonfactory.NewAddOnDeloymentConfigGetter(addonClient),
+				addonfactory.NewAddOnDeploymentConfigGetter(addonClient),
 				addonfactory.ToAddOnDeloymentConfigValues,
 			),
 		).
-		WithAgentRegistrationOption(manager.NewRegistrationOption(nativeClient))
+		WithAgentRegistrationOption(manager.NewRegistrationOption(nativeClient)).
+		WithAgentDeployTriggerClusterFilter(utils.ClusterImageRegistriesAnnotationChanged)
 
 	if agentInstallAll {
 		agentFactory.WithInstallStrategy(agent.InstallAllStrategy(common.AddonAgentInstallNamespace))

deploy/bases/addontemplate.yaml

@@ -45,7 +45,7 @@ spec:
                   serviceAccount: managed-serviceaccount
                   containers:
                     - name: addon-agent
-                      image: quay.io/open-cluster-management/managed-serviceaccount
+                      image: quay.io/open-cluster-management/managed-serviceaccount:latest
                       imagePullPolicy: IfNotPresent
                       command:
                         - /agent
@@ -60,10 +60,6 @@ spec:
                           port: 8000
                         initialDelaySeconds: 2
                         periodSeconds: 10
-            # {{ if .ImagePullSecretData }}
-            #       imagePullSecrets:
-            #       - name: open-cluster-management-image-pull-credentials
-            # {{- end }}
           - apiVersion: rbac.authorization.k8s.io/v1
             kind: Role
             metadata:
@@ -100,6 +96,8 @@ spec:
             metadata:
               name: managed-serviceaccount
               namespace: open-cluster-management-agent-addon
+            imagePullSecrets:
+              - name: open-cluster-management-image-pull-credentials
   registration:
     - type: KubeClient
       kubeClient:

e2e/ephemeral_identity/ephemeral_identity.go

@@ -54,6 +54,7 @@ var _ = Describe("Ephemeral ManagedServiceAccount Test", Label("ephemeral"), fun
 			Namespace: f.TestClusterName(),
 			Name:      targetName,
 		}, latest)
+		Expect(err).NotTo(HaveOccurred())
 
 		createdAt := latest.CreationTimestamp
 		deletedAt := latest.DeletionTimestamp

e2e/install/install.go

@@ -2,11 +2,13 @@ package install
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -16,6 +18,7 @@ import (
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/types"
 	addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
+	clusterv1 "open-cluster-management.io/api/cluster/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"open-cluster-management.io/managed-serviceaccount/e2e/framework"
@@ -45,7 +48,7 @@ var _ = Describe("Addon Installation Test", Label("install"),
 			tolerations := []corev1.Toleration{{Key: "node-role.kubernetes.io/infra", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}}
 
 			c := f.HubRuntimeClient()
-			By("Prepare a AddOnDeployMentConfig for managed-serviceaccount addon")
+			By("Prepare a AddOnDeploymentConfig for managed-serviceaccount addon")
 			Eventually(func() error {
 				err := c.Create(context.TODO(), &addonv1alpha1.AddOnDeploymentConfig{
 					ObjectMeta: metav1.ObjectMeta{
@@ -63,10 +66,8 @@ var _ = Describe("Addon Installation Test", Label("install"),
 				if errors.IsAlreadyExists(err) {
 					return nil
 				}
-				if err != nil {
-					return err
-				}
-				return nil
+
+				return err
 			}).WithTimeout(time.Minute).ShouldNot(HaveOccurred())
 
 			By("Add the config to managed-serviceaccount addon")
@@ -174,4 +175,107 @@ var _ = Describe("Addon Installation Test", Label("install"),
 				return nil
 			}).WithTimeout(time.Minute).ShouldNot(HaveOccurred())
 		})
+
+		It("Agent image should be overrode by cluster annotation", func() {
+			By("Get Addon agent install namespace")
+			addon := &addonv1alpha1.ManagedClusterAddOn{}
+			err := f.HubRuntimeClient().Get(context.TODO(), types.NamespacedName{
+				Namespace: f.TestClusterName(),
+				Name:      common.AddonName,
+			}, addon)
+			Expect(err).NotTo(HaveOccurred())
+			addonInstallNamespace := addon.Spec.InstallNamespace
+
+			By("Prepare cluster annotation for addon image override config")
+			overrideRegistries := addonv1alpha1.AddOnDeploymentConfigSpec{
+				Registries: []addonv1alpha1.ImageMirror{
+					{
+						Source: "quay.io/open-cluster-management",
+						Mirror: "quay.io/ocm",
+					},
+				},
+			}
+			registriesJson, err := json.Marshal(overrideRegistries)
+			Expect(err).ToNot(HaveOccurred())
+			Eventually(func() error {
+				cluster := &clusterv1.ManagedCluster{}
+				err := f.HubRuntimeClient().Get(context.Background(),
+					types.NamespacedName{Name: f.TestClusterName()},
+					cluster)
+				if err != nil {
+					return err
+				}
+
+				newCluster := cluster.DeepCopy()
+
+				annotations := cluster.Annotations
+				if annotations == nil {
+					annotations = make(map[string]string)
+				}
+				annotations[clusterv1.ClusterImageRegistriesAnnotationKey] = string(registriesJson)
+
+				newCluster.Annotations = annotations
+				return f.HubRuntimeClient().Update(context.Background(), newCluster)
+			}).WithTimeout(time.Minute).ShouldNot(HaveOccurred())
+
+			By("Make sure addon is configured")
+			Eventually(func() error {
+				agentDeploy := &appsv1.Deployment{}
+				agentDeploy, err := f.HubNativeClient().AppsV1().Deployments(addonInstallNamespace).Get(
+					context.Background(), "managed-serviceaccount-addon-agent", metav1.GetOptions{})
+				if err != nil {
+					return err
+				}
+
+				containers := agentDeploy.Spec.Template.Spec.Containers
+				if len(containers) != 1 {
+					return fmt.Errorf("expect one container, but %v", containers)
+				}
+
+				if containers[0].Image != "quay.io/ocm/managed-serviceaccount:latest" {
+					return fmt.Errorf("unexpected image %s", containers[0].Image)
+				}
+
+				return nil
+			}).WithTimeout(time.Minute).ShouldNot(HaveOccurred())
+
+			// restore the image override config, because the override image is not available
+			// but it is needed by the pre-delete job
+			By("Restore the managed cluster annotation")
+			Eventually(func() error {
+				cluster := &clusterv1.ManagedCluster{}
+				err := f.HubRuntimeClient().Get(context.Background(),
+					types.NamespacedName{Name: f.TestClusterName()},
+					cluster)
+				if err != nil {
+					return err
+				}
+
+				newCluster := cluster.DeepCopy()
+				delete(newCluster.Annotations, clusterv1.ClusterImageRegistriesAnnotationKey)
+				return f.HubRuntimeClient().Update(context.Background(), newCluster)
+			}).WithTimeout(time.Minute).ShouldNot(HaveOccurred())
+
+			By("Make sure addon config is restored")
+			Eventually(func() error {
+				agentDeploy := &appsv1.Deployment{}
+				agentDeploy, err := f.HubNativeClient().AppsV1().Deployments(addonInstallNamespace).Get(
+					context.Background(), "managed-serviceaccount-addon-agent", metav1.GetOptions{})
+				if err != nil {
+					return err
+				}
+
+				containers := agentDeploy.Spec.Template.Spec.Containers
+				if len(containers) != 1 {
+					return fmt.Errorf("expect one container, but %v", containers)
+				}
+
+				if containers[0].Image != "quay.io/open-cluster-management/managed-serviceaccount:latest" {
+					return fmt.Errorf("unexpected image %s", containers[0].Image)
+				}
+
+				return nil
+			}).WithTimeout(time.Minute).ShouldNot(HaveOccurred())
+		})
+
 	})

go.mod

@@ -13,8 +13,8 @@ require (
 	k8s.io/component-base v0.26.3
 	k8s.io/klog/v2 v2.80.1
 	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
-	open-cluster-management.io/addon-framework v0.6.1
-	open-cluster-management.io/api v0.11.0
+	open-cluster-management.io/addon-framework v0.7.1-0.20230803002113-eea76f8f5ad7
+	open-cluster-management.io/api v0.11.1-0.20230727093131-915f5826cff9
 	sigs.k8s.io/controller-runtime v0.14.5
 )
 

go.sum

@@ -677,10 +677,10 @@ k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+O
 k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
 k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y=
 k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-open-cluster-management.io/addon-framework v0.6.1 h1:gnBZaCRgtiPRjCBJoaRqMivajng/XOKp0NQhJUqLd+U=
-open-cluster-management.io/addon-framework v0.6.1/go.mod h1:Uu4XC3Ec0ATS7U73PJtzAP4NCDfbDBVy1k5RUUwQDqY=
-open-cluster-management.io/api v0.11.0 h1:zBxa33Co3wseLBF4HEJobhl0P6ygj+Drhe7Wrfo0/h8=
-open-cluster-management.io/api v0.11.0/go.mod h1:WgKUCJ7+Bf40DsOmH1Gdkpyj3joco+QLzrlM6Ak39zE=
+open-cluster-management.io/addon-framework v0.7.1-0.20230803002113-eea76f8f5ad7 h1:GozqvUtekO4Zg90wLih3CA/ZYKLJlGKX+vL990U8DoQ=
+open-cluster-management.io/addon-framework v0.7.1-0.20230803002113-eea76f8f5ad7/go.mod h1:gLGpXkdwAzzV+JB5eQPNHbZFJwp7HsKSSwgqOxGNVCw=
+open-cluster-management.io/api v0.11.1-0.20230727093131-915f5826cff9 h1:P5yjl8w09JYsTE1D6JV6y1vY9X2bBN8m494ZYg9HoyY=
+open-cluster-management.io/api v0.11.1-0.20230727093131-915f5826cff9/go.mod h1:WgKUCJ7+Bf40DsOmH1Gdkpyj3joco+QLzrlM6Ak39zE=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=