✨ Support override addon images by the cluster's annotation

Signed-off-by: zhujian <jiazhu@redhat.com>
open-cluster-management-io · Jul 26, 2023 · 9ace1a6 · 9ace1a6
1 parent ca59f26
commit 9ace1a6
Show file tree

Hide file tree

Showing 4 changed files with 336 additions and 29 deletions.
diff --git a/cmd/example/helloworld_helm/main.go b/cmd/example/helloworld_helm/main.go
@@ -107,12 +107,12 @@ func runController(ctx context.Context, kubeConfig *rest.Config) error {
 		).
 		WithGetValuesFuncs(
 			helloworld_helm.GetDefaultValues,
+			addonfactory.ToImageOverrideValuesFromClusterAnnotationFunc(
+				"global.imageOverrides.helloWorldHelm",
+				helloworld.DefaultHelloWorldExampleImage),
 			addonfactory.GetAddOnDeploymentConfigValues(
 				addonfactory.NewAddOnDeploymentConfigGetter(addonClient),
 				addonfactory.ToAddOnNodePlacementValues,
-				addonfactory.ToImageOverrideValuesFunc(
-					"global.imageOverrides.helloWorldHelm",
-					helloworld.DefaultHelloWorldExampleImage),
 			),
 			helloworld_helm.GetImageValues(kubeClient),
 			addonfactory.GetValuesFromAddonAnnotation,

diff --git a/pkg/addonfactory/addondeploymentconfig.go b/pkg/addonfactory/addondeploymentconfig.go
@@ -2,11 +2,13 @@ package addonfactory
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/klog/v2"
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
 	addonv1alpha1client "open-cluster-management.io/api/client/addon/clientset/versioned"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
@@ -229,37 +231,97 @@ func ToAddOnDeploymentConfigValues(config addonapiv1alpha1.AddOnDeploymentConfig
 // the imageKey is "helloWorldImage", the image is "quay.io/open-cluster-management/addon-agent:v1"
 // after transformed, the Values object will be: {"helloWorldImage": "quay.io/ocm/addon-agent:v1"}
 //
-// Note: the imageKey can support the nested key, for example: "global.imageOverrides.helloWorldImage", the output
-// will be: {"global": {"imageOverrides": {"helloWorldImage": "quay.io/ocm/addon-agent:v1"}}}
+// Note:
+//   - the imageKey can support the nested key, for example: "global.imageOverrides.helloWorldImage", the output
+//     will be: {"global": {"imageOverrides": {"helloWorldImage": "quay.io/ocm/addon-agent:v1"}}}
+//   - ToImageOverrideValuesFunc and ToImageOverrideValuesFromClusterAnnotationFunc are mutually exclusive,
+//     only one of them can be used by the same addon
 func ToImageOverrideValuesFunc(imageKey, image string) AddOnDeploymentConfigToValuesFunc {
 	return func(config addonapiv1alpha1.AddOnDeploymentConfig) (Values, error) {
-		if len(imageKey) == 0 {
-			return nil, fmt.Errorf("imageKey is empty")
-		}
-		if len(image) == 0 {
-			return nil, fmt.Errorf("image is empty")
+		getRegistries := func() ([]addonapiv1alpha1.ImageMirror, error) {
+			return config.Spec.Registries, nil
 		}
+		return overrideImageWithKeyValue(imageKey, image, getRegistries)
+	}
+}
+
+func overrideImageWithKeyValue(imageKey, image string, getRegistries func() ([]addonapiv1alpha1.ImageMirror, error),
+) (Values, error) {
+	if len(imageKey) == 0 {
+		return nil, fmt.Errorf("imageKey is empty")
+	}
+	if len(image) == 0 {
+		return nil, fmt.Errorf("image is empty")
+	}
 
-		nestedMap := make(map[string]interface{})
+	nestedMap := make(map[string]interface{})
 
-		keys := strings.Split(imageKey, ".")
-		currentMap := nestedMap
+	keys := strings.Split(imageKey, ".")
+	currentMap := nestedMap
 
-		for i := 0; i < len(keys)-1; i++ {
-			key := keys[i]
-			nextMap := make(map[string]interface{})
-			currentMap[key] = nextMap
-			currentMap = nextMap
-		}
+	for i := 0; i < len(keys)-1; i++ {
+		key := keys[i]
+		nextMap := make(map[string]interface{})
+		currentMap[key] = nextMap
+		currentMap = nextMap
+	}
 
-		lastKey := keys[len(keys)-1]
-		currentMap[lastKey] = image
+	lastKey := keys[len(keys)-1]
+	currentMap[lastKey] = image
+
+	registries, err := getRegistries()
+	if err != nil {
+		klog.Errorf("failed to get image registries, err %v", err)
+		return nestedMap, err
+	}
+
+	klog.V(4).Infof("Image registries values %v", registries)
+	if registries != nil {
+		currentMap[lastKey] = OverrideImage(registries, image)
+	}
+
+	return nestedMap, nil
+}
 
-		if config.Spec.Registries != nil {
-			currentMap[lastKey] = OverrideImage(config.Spec.Registries, image)
+const ClusterImageRegistriesAnnotation = "open-cluster-management.io/image-registries"
+
+// ToImageOverrideValuesFromClusterAnnotationFunc return a func that can use the registries configed by the annotation
+// "open-cluster-management.io/image-registries" on the managed cluster resource to override image.
+// then return the overridden value with key imageKey.
+//
+// for example: the annotation on the managed cluster resource is:
+// open-cluster-management.io/image-registries: '{"registries":[{"mirror":"quay.io/ocm","source":"quay.io/open-cluster-management"}]}'
+// the imageKey is "helloWorldImage", the image is "quay.io/open-cluster-management/addon-agent:v1"
+// after transformed, the Values object will be: {"helloWorldImage": "quay.io/ocm/addon-agent:v1"}
+//
+// Note:
+//   - the imageKey can support the nested key, for example: "global.imageOverrides.helloWorldImage", the output
+//     will be: {"global": {"imageOverrides": {"helloWorldImage": "quay.io/ocm/addon-agent:v1"}}}
+//   - ToImageOverrideValuesFromClusterAnnotationFunc and ToImageOverrideValuesFunc are mutually exclusive,
+//     only one of them can be used by the same addon
+func ToImageOverrideValuesFromClusterAnnotationFunc(imageKey, image string) GetValuesFunc {
+	return func(cluster *clusterv1.ManagedCluster, addon *addonapiv1alpha1.ManagedClusterAddOn) (Values, error) {
+
+		getRegistries := func() ([]addonapiv1alpha1.ImageMirror, error) {
+			annotations := cluster.GetAnnotations()
+			klog.V(4).Infof("Try to get image registries from annotation %v", annotations[ClusterImageRegistriesAnnotation])
+			if len(annotations[ClusterImageRegistriesAnnotation]) == 0 {
+				return nil, nil
+			}
+			type ImageRegistries struct {
+				Registries []addonapiv1alpha1.ImageMirror `json:"registries"`
+			}
+
+			imageRegistries := ImageRegistries{}
+			err := json.Unmarshal([]byte(annotations[ClusterImageRegistriesAnnotation]), &imageRegistries)
+			if err != nil {
+				klog.Errorf("failed to unmarshal the annotation %v, err %v", annotations[ClusterImageRegistriesAnnotation], err)
+				return nil, err
+			}
+			return imageRegistries.Registries, nil
 		}
 
-		return nestedMap, nil
+		return overrideImageWithKeyValue(imageKey, image, getRegistries)
 	}
 }
 

diff --git a/pkg/addonfactory/addondeploymentconfig_test.go b/pkg/addonfactory/addondeploymentconfig_test.go
@@ -5,13 +5,15 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"open-cluster-management.io/addon-framework/pkg/addonmanager/addontesting"
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
 	fakeaddon "open-cluster-management.io/api/client/addon/clientset/versioned/fake"
+	clusterv1 "open-cluster-management.io/api/cluster/v1"
 )
 
 var (
@@ -368,3 +370,143 @@ func TestToImageOverrideValuesFunc(t *testing.T) {
 		})
 	}
 }
+
+func TestToImageOverrideValuesFromClusterAnnotationFunc(t *testing.T) {
+	cases := []struct {
+		name           string
+		imageKey       string
+		imageValue     string
+		cluster        *clusterv1.ManagedCluster
+		expectedValues Values
+		expectedError  string
+	}{
+		{
+			name:       "no nested imagekey",
+			imageKey:   "image",
+			imageValue: "a/b/c:v1",
+			cluster: &clusterv1.ManagedCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						ClusterImageRegistriesAnnotation: `{"registries":[{"mirror":"x/y","source":"a/b"}]}`,
+					},
+				},
+			},
+			expectedValues: Values{
+				"image": "x/y/c:v1",
+			},
+		},
+		{
+			name:       "nested imagekey",
+			imageKey:   "global.imageOverride.image",
+			imageValue: "a/b/c:v1",
+			cluster: &clusterv1.ManagedCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						ClusterImageRegistriesAnnotation: `{"registries":[{"mirror":"x","source":"a"}]}`,
+					},
+				},
+			},
+			expectedValues: Values{
+				"global": map[string]interface{}{
+					"imageOverride": map[string]interface{}{
+						"image": "x/b/c:v1",
+					},
+				},
+			},
+		},
+		{
+			name:       "empty imagekey",
+			imageKey:   "",
+			imageValue: "a/b/c:v1",
+			cluster: &clusterv1.ManagedCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						ClusterImageRegistriesAnnotation: `{"registries":[{"mirror":"x","source":"a"}]}`,
+					},
+				},
+			},
+			expectedError: "imageKey is empty",
+		},
+		{
+			name:       "empty image",
+			imageKey:   "global.imageOverride.image",
+			imageValue: "",
+			cluster: &clusterv1.ManagedCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						ClusterImageRegistriesAnnotation: `{"registries":[{"mirror":"x","source":"a"}]}`,
+					},
+				},
+			},
+			expectedError: "image is empty",
+		},
+		{
+			name:       "source not match",
+			imageKey:   "global.imageOverride.image",
+			imageValue: "a/b/c",
+			cluster: &clusterv1.ManagedCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						ClusterImageRegistriesAnnotation: `{"registries":[{"mirror":"x","source":"b"}]}`,
+					},
+				},
+			},
+			expectedValues: Values{
+				"global": map[string]interface{}{
+					"imageOverride": map[string]interface{}{
+						"image": "a/b/c",
+					},
+				},
+			},
+		},
+		{
+			name:       "source empty",
+			imageKey:   "global.imageOverride.image",
+			imageValue: "a/b/c:v1",
+			cluster: &clusterv1.ManagedCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						ClusterImageRegistriesAnnotation: `{"registries":[{"mirror":"y"}]}`,
+					},
+				},
+			},
+			expectedValues: Values{
+				"global": map[string]interface{}{
+					"imageOverride": map[string]interface{}{
+						"image": "y/c:v1",
+					},
+				},
+			},
+		},
+		{
+			name:       "annotation invalid",
+			imageKey:   "image",
+			imageValue: "a/b/c:v1",
+			cluster: &clusterv1.ManagedCluster{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						ClusterImageRegistriesAnnotation: `{"registries":`,
+					},
+				},
+			},
+			expectedValues: Values{
+				"image": "a/b/c:v1",
+			},
+			expectedError: "unexpected end of JSON input",
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+
+			values, err := ToImageOverrideValuesFromClusterAnnotationFunc(c.imageKey, c.imageValue)(c.cluster, nil)
+			if err != nil || len(c.expectedError) > 0 {
+				assert.ErrorContains(t, err, c.expectedError, "expected error: %v, got: %v", c.expectedError, err)
+			}
+
+			if !equality.Semantic.DeepEqual(values, c.expectedValues) {
+				t.Errorf("expected values %v, but got values %v", c.expectedValues, values)
+			}
+		})
+	}
+}