✨ Support override addon images by the cluster's annotation

cmd/example/helloworld_helm/main.go

@@ -110,9 +110,11 @@ func runController(ctx context.Context, kubeConfig *rest.Config) error {
 			addonfactory.GetAddOnDeploymentConfigValues(
 				addonfactory.NewAddOnDeploymentConfigGetter(addonClient),
 				addonfactory.ToAddOnNodePlacementValues,
-				addonfactory.ToImageOverrideValuesFunc(
-					"global.imageOverrides.helloWorldHelm",
-					helloworld.DefaultHelloWorldExampleImage),
+			),
+			addonfactory.GetAgentImageValues(
+				addonfactory.NewAddOnDeploymentConfigGetter(addonClient),
+				"global.imageOverrides.helloWorldHelm",
+				helloworld.DefaultHelloWorldExampleImage,
 			),
 			helloworld_helm.GetImageValues(kubeClient),
 			addonfactory.GetValuesFromAddonAnnotation,

go.mod

@@ -21,7 +21,7 @@ require (
 	k8s.io/component-base v0.26.1
 	k8s.io/klog/v2 v2.80.1
 	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
-	open-cluster-management.io/api v0.11.1-0.20230609103311-088e8fe86139
+	open-cluster-management.io/api v0.11.1-0.20230727093131-915f5826cff9
 	sigs.k8s.io/controller-runtime v0.14.4
 )
 

go.sum

@@ -819,8 +819,8 @@ k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 h1:+70TFaan3hfJzs+7VK2o+O
 k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4=
 k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 h1:KTgPnR10d5zhztWptI952TNtt/4u5h3IzDXkdIMuo2Y=
 k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-open-cluster-management.io/api v0.11.1-0.20230609103311-088e8fe86139 h1:nw/XSv4eDGqmg0ks2PHzrE2uosvjw+D314843G56xGY=
-open-cluster-management.io/api v0.11.1-0.20230609103311-088e8fe86139/go.mod h1:WgKUCJ7+Bf40DsOmH1Gdkpyj3joco+QLzrlM6Ak39zE=
+open-cluster-management.io/api v0.11.1-0.20230727093131-915f5826cff9 h1:P5yjl8w09JYsTE1D6JV6y1vY9X2bBN8m494ZYg9HoyY=
+open-cluster-management.io/api v0.11.1-0.20230727093131-915f5826cff9/go.mod h1:WgKUCJ7+Bf40DsOmH1Gdkpyj3joco+QLzrlM6Ak39zE=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

pkg/addonfactory/addondeploymentconfig.go

@@ -2,18 +2,28 @@ package addonfactory
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/klog/v2"
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
 	addonv1alpha1client "open-cluster-management.io/api/client/addon/clientset/versioned"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
 
 	"open-cluster-management.io/addon-framework/pkg/utils"
 )
 
+// Deprecated: use AddOnDeploymentConfigGVR in package "open-cluster-management.io/addon-framework/pkg/utils" instead.
+var AddOnDeploymentConfigGVR = schema.GroupVersionResource{
+	Group:    "addon.open-cluster-management.io",
+	Version:  "v1alpha1",
+	Resource: "addondeploymentconfigs",
+}
+
 // AddOnDeloymentConfigToValuesFunc transform the AddOnDeploymentConfig object into Values object
 // The transformation logic depends on the definition of the addon template
 // Deprecated: use AddOnDeploymentConfigToValuesFunc instead.
@@ -229,38 +239,149 @@ func ToAddOnDeploymentConfigValues(config addonapiv1alpha1.AddOnDeploymentConfig
 // the imageKey is "helloWorldImage", the image is "quay.io/open-cluster-management/addon-agent:v1"
 // after transformed, the Values object will be: {"helloWorldImage": "quay.io/ocm/addon-agent:v1"}
 //
-// Note: the imageKey can support the nested key, for example: "global.imageOverrides.helloWorldImage", the output
-// will be: {"global": {"imageOverrides": {"helloWorldImage": "quay.io/ocm/addon-agent:v1"}}}
+// Note:
+//   - the imageKey can support the nested key, for example: "global.imageOverrides.helloWorldImage", the output
+//     will be: {"global": {"imageOverrides": {"helloWorldImage": "quay.io/ocm/addon-agent:v1"}}}
+//   - If you want to override the image with the value from the AddOnDeploymentConfig.spec.Registries first, and
+//     if it is not changed, then override it with the value from the cluster annotation, you can use the function
+//     GetAgentImageValues instead.
 func ToImageOverrideValuesFunc(imageKey, image string) AddOnDeploymentConfigToValuesFunc {
 	return func(config addonapiv1alpha1.AddOnDeploymentConfig) (Values, error) {
-		if len(imageKey) == 0 {
-			return nil, fmt.Errorf("imageKey is empty")
+		values, _, err := overrideImageWithKeyValue(imageKey, image, getRegistriesFromAddonDeploymentConfig(config))
+		return values, err
+	}
+}
+
+func getRegistriesFromAddonDeploymentConfig(
+	config addonapiv1alpha1.AddOnDeploymentConfig) func() ([]addonapiv1alpha1.ImageMirror, error) {
+	return func() ([]addonapiv1alpha1.ImageMirror, error) {
+		return config.Spec.Registries, nil
+	}
+}
+
+func getRegistriesFromClusterAnnotation(
+	cluster *clusterv1.ManagedCluster) func() ([]addonapiv1alpha1.ImageMirror, error) {
+	return func() ([]addonapiv1alpha1.ImageMirror, error) {
+		if cluster == nil {
+			return nil, nil
+		}
+		annotations := cluster.GetAnnotations()
+		klog.V(4).Infof("Try to get image registries from annotation %v", annotations[clusterv1.ClusterImageRegistriesAnnotationKey])
+		if len(annotations[clusterv1.ClusterImageRegistriesAnnotationKey]) == 0 {
+			return nil, nil
 		}
-		if len(image) == 0 {
-			return nil, fmt.Errorf("image is empty")
+		type ImageRegistries struct {
+			Registries []addonapiv1alpha1.ImageMirror `json:"registries"`
 		}
 
-		nestedMap := make(map[string]interface{})
+		imageRegistries := ImageRegistries{}
+		err := json.Unmarshal([]byte(annotations[clusterv1.ClusterImageRegistriesAnnotationKey]), &imageRegistries)
+		if err != nil {
+			klog.Errorf("failed to unmarshal the annotation %v, err %v", annotations[clusterv1.ClusterImageRegistriesAnnotationKey], err)
+			return nil, err
+		}
+		return imageRegistries.Registries, nil
+	}
+}
+
+// GetAgentImageValues return a func that can use two ways, AddOnDeploymentConfig.spec.Registries and annotation on the
+// ManagedCluster, to override image, then return the overridden value with key imageKey.
+// For example:
+//
+//  1. configure the image registries with the spec of the AddOnDeploymentConfig, if the registries in the spec is:
+//     {...,"spec":{"registries":[{"mirror":"quay.io/ocm/addon-agent","source":"quay.io/open-cluster-management/addon-agent"}]}}
+//     the "imageKey" is "helloWorldImage", the "image" is "quay.io/open-cluster-management/addon-agent:v1"
+//     after transformed, the Values object will be: {"helloWorldImage": "quay.io/ocm/addon-agent:v1"}
+//
+//  2. configure the image registries with the annotation "open-cluster-management.io/image-registries" on the
+//     ManagedCluster, if the annotation on the managed cluster resource is:
+//     "open-cluster-management.io/image-registries": '{"registries":[{"mirror":"quay.io/ocm","source":"quay.io/open-cluster-management"}]}'
+//     the "imageKey" is "helloWorldImage", the "image" is "quay.io/open-cluster-management/addon-agent:v1"
+//     after transformed, the Values object will be: {"helloWorldImage": "quay.io/ocm/addon-agent:v1"}
+//
+// Note:
+//   - the imageKey can support the nested key, for example: "global.imageOverrides.helloWorldImage", the output
+//     will be: {"global": {"imageOverrides": {"helloWorldImage": "quay.io/ocm/addon-agent:v1"}}}
+//   - Image registries configured in the addonDeploymentConfig will take precedence over the managed cluster annotation
+func GetAgentImageValues(getter AddOnDeploymentConfigGetter, imageKey, image string) GetValuesFunc {
+	return func(cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn) (Values, error) {
 
-		keys := strings.Split(imageKey, ".")
-		currentMap := nestedMap
+		// Get image from AddOnDeploymentConfig
+		for _, config := range addon.Status.ConfigReferences {
+			if config.ConfigGroupResource.Group != utils.AddOnDeploymentConfigGVR.Group ||
+				config.ConfigGroupResource.Resource != utils.AddOnDeploymentConfigGVR.Resource {
+				continue
+			}
 
-		for i := 0; i < len(keys)-1; i++ {
-			key := keys[i]
-			nextMap := make(map[string]interface{})
-			currentMap[key] = nextMap
-			currentMap = nextMap
-		}
+			addOnDeploymentConfig, err := getter.Get(context.Background(), config.Namespace, config.Name)
+			if err != nil {
+				return nil, err
+			}
+
+			values, overrode, err := overrideImageWithKeyValue(imageKey, image,
+				getRegistriesFromAddonDeploymentConfig(*addOnDeploymentConfig))
+			if err != nil {
+				return values, err
+			}
 
-		lastKey := keys[len(keys)-1]
-		currentMap[lastKey] = image
+			// if the image is overrode by AddOnDeploymentConfig, use the overrode value
+			if overrode {
+				klog.V(4).Infof("Overrode image %v with %v", image, values)
+				return values, nil
+			}
+		}
 
-		if config.Spec.Registries != nil {
-			currentMap[lastKey] = OverrideImage(config.Spec.Registries, image)
+		// If the image is not overrode by AddOnDeploymentConfig, try to get image from cluster annotation
+		values, _, err := overrideImageWithKeyValue(imageKey, image, getRegistriesFromClusterAnnotation(cluster))
+		if err != nil {
+			return values, err
 		}
+		return values, nil
+	}
+}
+
+func overrideImageWithKeyValue(imageKey, image string, getRegistries func() ([]addonapiv1alpha1.ImageMirror, error),
+) (Values, bool, error) {
+
+	if len(imageKey) == 0 {
+		return nil, false, fmt.Errorf("imageKey is empty")
+	}
+	if len(image) == 0 {
+		return nil, false, fmt.Errorf("image is empty")
+	}
+
+	nestedMap := make(map[string]interface{})
+
+	keys := strings.Split(imageKey, ".")
+	currentMap := nestedMap
+
+	for i := 0; i < len(keys)-1; i++ {
+		key := keys[i]
+		nextMap := make(map[string]interface{})
+		currentMap[key] = nextMap
+		currentMap = nextMap
+	}
 
-		return nestedMap, nil
+	lastKey := keys[len(keys)-1]
+	currentMap[lastKey] = image
+
+	registries, err := getRegistries()
+	if err != nil {
+		klog.Errorf("failed to get image registries, err %v", err)
+		return nestedMap, false, err
+	}
+
+	overrode := false
+	klog.V(4).Infof("Image registries values %v", registries)
+	if registries != nil {
+		overrodeImage := OverrideImage(registries, image)
+		currentMap[lastKey] = overrodeImage
+		if overrodeImage != image {
+			overrode = true
+		}
 	}
+
+	return nestedMap, overrode, nil
 }
 
 // OverrideImage checks whether the source configured in registries can match the imagedName, if yes will use the