Support daemonSet with AddOnTemplate

Signed-off-by: zhujian <jiazhu@redhat.com>

go.mod

@@ -34,7 +34,7 @@ require (
 	k8s.io/klog/v2 v2.120.1
 	k8s.io/kube-aggregator v0.29.3
 	k8s.io/utils v0.0.0-20240310230437-4693a0247e57
-	open-cluster-management.io/addon-framework v0.9.1-0.20240419070222-e703fc5a2556
+	open-cluster-management.io/addon-framework v0.9.1-0.20240617015910-660e21bcfd3b
 	open-cluster-management.io/api v0.13.1-0.20240605083248-f9e7f50520fc
 	open-cluster-management.io/sdk-go v0.13.1-0.20240618022514-b2c1dd175afd
 	sigs.k8s.io/controller-runtime v0.17.3

go.sum

@@ -465,8 +465,8 @@ k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/A
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA=
 k8s.io/utils v0.0.0-20240310230437-4693a0247e57 h1:gbqbevonBh57eILzModw6mrkbwM0gQBEuevE/AaBsHY=
 k8s.io/utils v0.0.0-20240310230437-4693a0247e57/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-open-cluster-management.io/addon-framework v0.9.1-0.20240419070222-e703fc5a2556 h1:X3vJEx9agC94l7SitpWZFDshISdL1niqVH0+diyqfJo=
-open-cluster-management.io/addon-framework v0.9.1-0.20240419070222-e703fc5a2556/go.mod h1:HayKCznnlyW+0dUJQGj5sNR6i3tvylSySD3YnvZkBtY=
+open-cluster-management.io/addon-framework v0.9.1-0.20240617015910-660e21bcfd3b h1:6M9IdEa71Qegl5HjaRcsuLAZvimIafweix2aUBzwot0=
+open-cluster-management.io/addon-framework v0.9.1-0.20240617015910-660e21bcfd3b/go.mod h1:HayKCznnlyW+0dUJQGj5sNR6i3tvylSySD3YnvZkBtY=
 open-cluster-management.io/api v0.13.1-0.20240605083248-f9e7f50520fc h1:tcfncubZRFphYtDXBE7ApBNlSnj1RNazhW+8F01XYYg=
 open-cluster-management.io/api v0.13.1-0.20240605083248-f9e7f50520fc/go.mod h1:ltijKJhDifrPH0csvCUmFt5lzaERv+BBfh6X3l83rT0=
 open-cluster-management.io/sdk-go v0.13.1-0.20240618022514-b2c1dd175afd h1:kTVZOR7bTdh4ID7EoliyGhPR5CItpx8GehN581IxoPA=

pkg/addon/templateagent/decorator.go

@@ -125,6 +125,48 @@ func (d *deploymentDecorator) decorate(obj *unstructured.Unstructured) (*unstruc
 	return &unstructured.Unstructured{Object: result}, nil
 }
 
+type daemonSetDecorator struct {
+	decorators []podTemplateSpecDecorator
+}
+
+func newDaemonSetDecorator(
+	addonName string,
+	template *addonapiv1alpha1.AddOnTemplate,
+	orderedValues orderedValues,
+	privateValues addonfactory.Values,
+) decorator {
+	return &daemonSetDecorator{
+		decorators: []podTemplateSpecDecorator{
+			newEnvironmentDecorator(orderedValues),
+			newVolumeDecorator(addonName, template),
+			newNodePlacementDecorator(privateValues),
+			newImageDecorator(privateValues),
+		},
+	}
+}
+
+func (d *daemonSetDecorator) decorate(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) {
+	daemonSet, err := utils.ConvertToDaemonSet(obj)
+	// not a deployment, directly return
+	if err != nil {
+		return obj, nil
+	}
+
+	for _, decorator := range d.decorators {
+		err = decorator.decorate(&daemonSet.Spec.Template)
+		if err != nil {
+			return obj, err
+		}
+	}
+
+	result, err := runtime.DefaultUnstructuredConverter.ToUnstructured(daemonSet)
+	if err != nil {
+		return obj, err
+	}
+
+	return &unstructured.Unstructured{Object: result}, nil
+}
+
 type podTemplateSpecDecorator interface {
 	// decorate modifies the deployment in place
 	decorate(pod *corev1.PodTemplateSpec) error

pkg/addon/templateagent/template_agent.go

@@ -116,7 +116,7 @@ func (a *CRDTemplateAgentAddon) GetAgentAddonOptions() agent.AgentAddonOptions {
 	agentAddonOptions := agent.AgentAddonOptions{
 		AddonName: a.addonName,
 		HealthProber: &agent.HealthProber{
-			Type: agent.HealthProberTypeDeploymentAvailability,
+			Type: agent.HealthProberTypeWorkloadAvailability,
 		},
 		SupportedConfigGVRs: supportedConfigGVRs,
 		Registration: &agent.RegistrationOption{
@@ -189,6 +189,7 @@ func (a *CRDTemplateAgentAddon) decorateObject(
 	privateValues addonfactory.Values) (*unstructured.Unstructured, error) {
 	decorators := []decorator{
 		newDeploymentDecorator(a.addonName, template, orderedValues, privateValues),
+		newDaemonSetDecorator(a.addonName, template, orderedValues, privateValues),
 		newNamespaceDecorator(privateValues),
 	}
 

pkg/addon/templateagent/template_agent_test.go

@@ -142,96 +142,112 @@ func TestAddonTemplateAgentManifests(t *testing.T) {
 			),
 			managedCluster: addonfactory.NewFakeManagedCluster(clusterName, "1.10.1"),
 			validateObjects: func(t *testing.T, objects []runtime.Object) {
-				if len(objects) != 4 {
-					t.Errorf("expected 4 objects, but got %v", len(objects))
+				if len(objects) != 5 {
+					t.Fatalf("expected 5 objects, but got %v", len(objects))
 				}
 
-				unstructObject, ok := objects[0].(*unstructured.Unstructured)
-				if !ok {
-					t.Errorf("expected object to be *appsv1.Deployment, but got %T", objects[0])
-				}
-				object, err := utils.ConvertToDeployment(unstructObject)
-				if err != nil {
-					t.Fatal(err)
-				}
-				image := object.Spec.Template.Spec.Containers[0].Image
-				if image != "quay.io/ocm/addon-examples:v1" {
-					t.Errorf("unexpected image %v", image)
-				}
-
-				if object.Namespace != "test-install-namespace" {
-					t.Errorf("unexpected namespace %s", object.Namespace)
-				}
+				validatePodTemplate := func(t *testing.T, podTemplate corev1.PodTemplateSpec) {
+					image := podTemplate.Spec.Containers[0].Image
+					if image != "quay.io/ocm/addon-examples:v1" {
+						t.Errorf("unexpected image %v", image)
+					}
+
+					nodeSelector := podTemplate.Spec.NodeSelector
+					expectedNodeSelector := map[string]string{"host": "ssd"}
+					if !equality.Semantic.DeepEqual(nodeSelector, expectedNodeSelector) {
+						t.Errorf("unexpected nodeSelector %v", nodeSelector)
+					}
+
+					tolerations := podTemplate.Spec.Tolerations
+					expectedTolerations := []corev1.Toleration{{Key: "foo", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute}}
+					if !equality.Semantic.DeepEqual(tolerations, expectedTolerations) {
+						t.Errorf("unexpected tolerations %v", tolerations)
+					}
+
+					envs := podTemplate.Spec.Containers[0].Env
+					expectedEnvs := []corev1.EnvVar{
+						{Name: "LOG_LEVEL", Value: "4"},
+						{Name: "HUB_KUBECONFIG", Value: "/managed/hub-kubeconfig/kubeconfig"},
+						{Name: "CLUSTER_NAME", Value: clusterName},
+						{Name: "INSTALL_NAMESPACE", Value: "test-install-namespace"},
+					}
+					if !equality.Semantic.DeepEqual(envs, expectedEnvs) {
+						t.Errorf("unexpected envs %v", envs)
+					}
+
+					volumes := podTemplate.Spec.Volumes
+					expectedVolumes := []corev1.Volume{
+						{
+							Name: "hub-kubeconfig",
+							VolumeSource: corev1.VolumeSource{
+								Secret: &corev1.SecretVolumeSource{
+									SecretName: "hello-hub-kubeconfig",
+								},
+							},
+						},
+						{
+							Name: "cert-example-com-signer-name",
+							VolumeSource: corev1.VolumeSource{
+								Secret: &corev1.SecretVolumeSource{
+									SecretName: "hello-example.com-signer-name-client-cert",
+								},
+							},
+						},
+					}
 
-				nodeSelector := object.Spec.Template.Spec.NodeSelector
-				expectedNodeSelector := map[string]string{"host": "ssd"}
-				if !equality.Semantic.DeepEqual(nodeSelector, expectedNodeSelector) {
-					t.Errorf("unexpected nodeSelector %v", nodeSelector)
-				}
+					if !equality.Semantic.DeepEqual(volumes, expectedVolumes) {
+						t.Errorf("expected volumes %v, but got: %v", expectedVolumes, volumes)
+					}
 
-				tolerations := object.Spec.Template.Spec.Tolerations
-				expectedTolerations := []corev1.Toleration{{Key: "foo", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute}}
-				if !equality.Semantic.DeepEqual(tolerations, expectedTolerations) {
-					t.Errorf("unexpected tolerations %v", tolerations)
+					volumeMounts := podTemplate.Spec.Containers[0].VolumeMounts
+					expectedVolumeMounts := []corev1.VolumeMount{
+						{
+							Name:      "hub-kubeconfig",
+							MountPath: "/managed/hub-kubeconfig",
+						},
+						{
+							Name:      "cert-example-com-signer-name",
+							MountPath: "/managed/example.com-signer-name",
+						},
+					}
+					if !equality.Semantic.DeepEqual(volumeMounts, expectedVolumeMounts) {
+						t.Errorf("expected volumeMounts %v, but got: %v", expectedVolumeMounts, volumeMounts)
+					}
 				}
 
-				envs := object.Spec.Template.Spec.Containers[0].Env
-				expectedEnvs := []corev1.EnvVar{
-					{Name: "LOG_LEVEL", Value: "4"},
-					{Name: "HUB_KUBECONFIG", Value: "/managed/hub-kubeconfig/kubeconfig"},
-					{Name: "CLUSTER_NAME", Value: clusterName},
-					{Name: "INSTALL_NAMESPACE", Value: "test-install-namespace"},
+				unstructDeployment, ok := objects[0].(*unstructured.Unstructured)
+				if !ok {
+					t.Errorf("expected object to be *appsv1.Deployment, but got %T", objects[0])
 				}
-				if !equality.Semantic.DeepEqual(envs, expectedEnvs) {
-					t.Errorf("unexpected envs %v", envs)
+				deployment, err := utils.ConvertToDeployment(unstructDeployment)
+				if err != nil {
+					t.Fatal(err)
 				}
-
-				volumes := object.Spec.Template.Spec.Volumes
-				expectedVolumes := []corev1.Volume{
-					{
-						Name: "hub-kubeconfig",
-						VolumeSource: corev1.VolumeSource{
-							Secret: &corev1.SecretVolumeSource{
-								SecretName: "hello-hub-kubeconfig",
-							},
-						},
-					},
-					{
-						Name: "cert-example-com-signer-name",
-						VolumeSource: corev1.VolumeSource{
-							Secret: &corev1.SecretVolumeSource{
-								SecretName: "hello-example.com-signer-name-client-cert",
-							},
-						},
-					},
+				if deployment.Namespace != "test-install-namespace" {
+					t.Errorf("unexpected namespace %s", deployment.Namespace)
 				}
+				validatePodTemplate(t, deployment.Spec.Template)
 
-				if !equality.Semantic.DeepEqual(volumes, expectedVolumes) {
-					t.Errorf("expected volumes %v, but got: %v", expectedVolumes, volumes)
+				unstructDaemonSet, ok := objects[1].(*unstructured.Unstructured)
+				if !ok {
+					t.Errorf("expected object to be *appsv1.DaemonSet, but got %T", objects[0])
 				}
-
-				volumeMounts := object.Spec.Template.Spec.Containers[0].VolumeMounts
-				expectedVolumeMounts := []corev1.VolumeMount{
-					{
-						Name:      "hub-kubeconfig",
-						MountPath: "/managed/hub-kubeconfig",
-					},
-					{
-						Name:      "cert-example-com-signer-name",
-						MountPath: "/managed/example.com-signer-name",
-					},
+				daemonSet, err := utils.ConvertToDaemonSet(unstructDaemonSet)
+				if err != nil {
+					t.Fatal(err)
 				}
-				if !equality.Semantic.DeepEqual(volumeMounts, expectedVolumeMounts) {
-					t.Errorf("expected volumeMounts %v, but got: %v", expectedVolumeMounts, volumeMounts)
+				if daemonSet.Namespace != "test-install-namespace" {
+					t.Errorf("unexpected namespace %s", daemonSet.Namespace)
 				}
+				validatePodTemplate(t, daemonSet.Spec.Template)
 
 				// check clusterrole
-				unstructObject, ok = objects[2].(*unstructured.Unstructured)
+				unstructCRB, ok := objects[3].(*unstructured.Unstructured)
 				if !ok {
 					t.Errorf("expected object to be unstructured, but got %T", objects[0])
 				}
 				clusterRoleBinding := &rbacv1.ClusterRoleBinding{}
-				err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructObject.Object, clusterRoleBinding)
+				err = runtime.DefaultUnstructuredConverter.FromUnstructured(unstructCRB.Object, clusterRoleBinding)
 				if err != nil {
 					t.Fatal(err)
 				}

pkg/addon/templateagent/testmanifests/addontemplate.yaml

@@ -41,6 +41,39 @@ spec:
                       env:
                         - name: LOG_LEVEL
                           value: "{{LOG_LEVEL}}" # addonDeploymentConfig variables
+          - kind: DaemonSet
+            apiVersion: apps/v1
+            metadata:
+              name: hello-template-agent-ds
+              namespace: open-cluster-management-agent-addon
+              annotations:
+                "addon.open-cluster-management.io/deletion-orphan": ""
+              labels:
+                app: hello-template-agent
+            spec:
+              selector:
+                matchLabels:
+                  app: hello-template-agent
+              template:
+                metadata:
+                  labels:
+                    app: hello-template-agent
+                spec:
+                  serviceAccountName: hello-template-agent-sa
+                  containers:
+                    - name: helloworld-agent
+                      image: quay.io/open-cluster-management/addon-examples:v1
+                      imagePullPolicy: IfNotPresent
+                      args:
+                        - "/helloworld_helm"
+                        - "agent"
+                        - "--cluster-name={{CLUSTER_NAME}}"
+                        - "--addon-namespace=open-cluster-management-agent-addon"
+                        - "--addon-name=hello-template"
+                        - "--hub-kubeconfig={{HUB_KUBECONFIG}}"
+                      env:
+                        - name: LOG_LEVEL
+                          value: "{{LOG_LEVEL}}" # addonDeploymentConfig variables
           - kind: ServiceAccount
             apiVersion: v1
             metadata:
@@ -113,4 +146,4 @@ spec:
           - o1
           - o2
           user: user1
-      type: CustomSigner
+      type: CustomSigner

test/e2e/addonmanagement_test.go

@@ -216,21 +216,30 @@ var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered,
 		manifestWork, err := t.HubWorkClient.WorkV1().ManifestWorks(clusterName).Get(context.Background(),
 			fmt.Sprintf("addon-%s-deploy-0", addOnName), metav1.GetOptions{})
 		gomega.Expect(err).ToNot(gomega.HaveOccurred())
-		foundConfig := false
-		expectedResourceIdentifier := workapiv1.ResourceIdentifier{
+		foundDeploymentConfig := false
+		expectedDeploymentResourceIdentifier := workapiv1.ResourceIdentifier{
 			Group:     "apps",
 			Resource:  "deployments",
 			Name:      "hello-template-agent",
 			Namespace: addonInstallNamespace,
 		}
+		foundDaemonSetConfig := false
+		expectedDaemonSetResourceIdentifier := workapiv1.ResourceIdentifier{
+			Group:     "apps",
+			Resource:  "daemonsets",
+			Name:      "hello-template-agent-ds",
+			Namespace: addonInstallNamespace,
+		}
 		for _, mc := range manifestWork.Spec.ManifestConfigs {
-			if mc.ResourceIdentifier == expectedResourceIdentifier {
-				foundConfig = true
+			if mc.ResourceIdentifier == expectedDeploymentResourceIdentifier {
+				foundDeploymentConfig = true
 				gomega.Expect(mc.UpdateStrategy.Type).To(gomega.Equal(workapiv1.UpdateStrategyTypeServerSideApply))
-				break
+			}
+			if mc.ResourceIdentifier == expectedDaemonSetResourceIdentifier {
+				foundDaemonSetConfig = true
 			}
 		}
-		if !foundConfig {
+		if !foundDeploymentConfig || !foundDaemonSetConfig {
 			gomega.Expect(fmt.Errorf("expected manifestwork is not correct, %v",
 				manifestWork.Spec.ManifestConfigs)).ToNot(gomega.HaveOccurred())
 		}
@@ -282,7 +291,8 @@ var _ = ginkgo.Describe("Enable addon management feature gate", ginkgo.Ordered,
 		}, eventuallyTimeout, eventuallyInterval).ShouldNot(gomega.HaveOccurred())
 	})
 
-	ginkgo.It("Template type addon should be configured by addon deployment config for image override even there are cluster annotation config", func() {
+	ginkgo.It("Template type addon should be configured by addon deployment config for image override"+
+		"even there are cluster annotation config", func() {
 		ginkgo.By("Prepare cluster annotation for addon image override config")
 		overrideRegistries := addonapiv1alpha1.AddOnDeploymentConfigSpec{
 			// should be different from the registries in the addonDeploymentConfig