Add a controller to support addon template (#187)

* Add addon template api Signed-off-by: zhujian <jiazhu@redhat.com> * Add addon template controller Signed-off-by: zhujian <jiazhu@redhat.com> * Add e2e for addon template Signed-off-by: zhujian <jiazhu@redhat.com> * Refactor addon template controller to start manager per cma Signed-off-by: zhujian <jiazhu@redhat.com> * Add unit tests for addon templates Signed-off-by: zhujian <jiazhu@redhat.com> * expose informrs when run addon manager Signed-off-by: zhujian <jiazhu@redhat.com> * remove unnecessary permissions Signed-off-by: zhujian <jiazhu@redhat.com> * get addon template from cache Signed-off-by: zhujian <jiazhu@redhat.com> * move template agent files to a seperate packate Signed-off-by: zhujian <jiazhu@redhat.com> * add indexers before informers start Signed-off-by: zhujian <jiazhu@redhat.com> * change template registration functions to methods Signed-off-by: zhujian <jiazhu@redhat.com> * add unit tests for addon template controller Signed-off-by: zhujian <jiazhu@redhat.com> * fix e2e issues Signed-off-by: zhujian <jiazhu@redhat.com> * add list watch rolebinding permission for addon manager Signed-off-by: zhujian <jiazhu@redhat.com> * update config spec hash by addon framework for template type addon Signed-off-by: zhujian <jiazhu@redhat.com> --------- Signed-off-by: zhujian <jiazhu@redhat.com>
open-cluster-management-io · Jun 21, 2023 · 0056b14 · 0056b14
1 parent ebc4aa8
commit 0056b14
Show file tree

Hide file tree

Showing 87 changed files with 5,926 additions and 391 deletions.
diff --git a/Makefile b/Makefile
@@ -92,6 +92,15 @@ deploy-helloworld-hosted: ensure-kustomize
 	$(KUSTOMIZE) build examples/deploy/addon/helloworld-hosted | $(KUBECTL) apply -f -
 	mv examples/deploy/addon/helloworld-hosted/kustomization.yaml.tmp examples/deploy/addon/helloworld-hosted/kustomization.yaml
 
+deploy-helloworld-template: ensure-kustomize
+	$(KUBECTL) create namespace $(MANAGED_CLUSTER_NAME) --dry-run=client -o yaml | $(KUBECTL) apply -f -
+# remove the following line when the registration-operator is supported to install the addon template CRD
+	$(KUBECTL) apply -f ./vendor/open-cluster-management.io/api/addon/v1alpha1/0000_03_addon.open-cluster-management.io_addontemplates.crd.yaml
+	cp examples/deploy/addon/helloworld-template/kustomization.yaml examples/deploy/addon/helloworld-template/kustomization.yaml.tmp
+	cd examples/deploy/addon/helloworld-template && ../../../../$(KUSTOMIZE) edit set image quay.io/open-cluster-management/addon-examples=$(EXAMPLE_IMAGE_NAME)
+	$(KUSTOMIZE) build examples/deploy/addon/helloworld-template | $(KUBECTL) apply -f -
+	mv examples/deploy/addon/helloworld-template/kustomization.yaml.tmp examples/deploy/addon/helloworld-template/kustomization.yaml
+
 undeploy-addon:
 	$(KUBECTL) delete -f examples/deploy/addon/helloworld-hosted/resources/helloworld_hosted_clustermanagementaddon.yaml --ignore-not-found
 	$(KUBECTL) delete -f examples/deploy/addon/helloworld-helm/resources/helloworld_helm_clustermanagementaddon.yaml --ignore-not-found
@@ -113,7 +122,7 @@ undeploy-helloworld-hosted: ensure-kustomize
 build-e2e:
 	go test -c ./test/e2e
 
-test-e2e: build-e2e deploy-ocm deploy-addon-manager deploy-helloworld deploy-helloworld-helm
+test-e2e: build-e2e deploy-ocm deploy-addon-manager deploy-helloworld deploy-helloworld-helm deploy-helloworld-template
 	./e2e.test -test.v -ginkgo.v
 
 build-hosted-e2e:
@@ -122,4 +131,4 @@ build-hosted-e2e:
 test-hosted-e2e: build-hosted-e2e deploy-hosted-ocm deploy-addon-manager deploy-helloworld-hosted
 	./e2ehosted.test -test.v -ginkgo.v
 
-include ./test/integration-test.mk
+include ./test/integration-test.mk
diff --git a/build/Dockerfile b/build/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.19 AS builder
+FROM golang:1.19-bullseye AS builder
 ARG OS=linux
 ARG ARCH=amd64
 WORKDIR /go/src/open-cluster-management.io/addon-framework

diff --git a/build/Dockerfile.example b/build/Dockerfile.example
@@ -1,4 +1,4 @@
-FROM golang:1.19 AS builder
+FROM golang:1.19-bullseye AS builder
 WORKDIR /go/src/open-cluster-management.io/addon-framework
 COPY . .
 ENV GO_PACKAGE open-cluster-management.io/addon-framework

diff --git a/cmd/example/helloworld/main.go b/cmd/example/helloworld/main.go
@@ -23,6 +23,7 @@ import (
 	"open-cluster-management.io/addon-framework/pkg/addonmanager"
 	addonagent "open-cluster-management.io/addon-framework/pkg/agent"
 	cmdfactory "open-cluster-management.io/addon-framework/pkg/cmd/factory"
+	"open-cluster-management.io/addon-framework/pkg/utils"
 	"open-cluster-management.io/addon-framework/pkg/version"
 )
 
@@ -93,7 +94,7 @@ func runController(ctx context.Context, kubeConfig *rest.Config) error {
 	)
 
 	agentAddon, err := addonfactory.NewAgentAddonFactory(helloworld.AddonName, helloworld.FS, "manifests/templates").
-		WithConfigGVRs(addonfactory.AddOnDeploymentConfigGVR).
+		WithConfigGVRs(utils.AddOnDeploymentConfigGVR).
 		WithGetValuesFuncs(
 			helloworld.GetDefaultValues,
 			addonfactory.GetAddOnDeploymentConfigValues(

diff --git a/cmd/example/helloworld_helm/main.go b/cmd/example/helloworld_helm/main.go
@@ -25,6 +25,7 @@ import (
 	"open-cluster-management.io/addon-framework/pkg/addonfactory"
 	"open-cluster-management.io/addon-framework/pkg/addonmanager"
 	cmdfactory "open-cluster-management.io/addon-framework/pkg/cmd/factory"
+	"open-cluster-management.io/addon-framework/pkg/utils"
 	"open-cluster-management.io/addon-framework/pkg/version"
 )
 
@@ -102,7 +103,7 @@ func runController(ctx context.Context, kubeConfig *rest.Config) error {
 	agentAddon, err := addonfactory.NewAgentAddonFactory(helloworld_helm.AddonName, helloworld_helm.FS, "manifests/charts/helloworld").
 		WithConfigGVRs(
 			schema.GroupVersionResource{Version: "v1", Resource: "configmaps"},
-			addonfactory.AddOnDeploymentConfigGVR,
+			utils.AddOnDeploymentConfigGVR,
 		).
 		WithGetValuesFuncs(
 			helloworld_helm.GetDefaultValues,

diff --git a/deploy/resources/cluster_role.yaml b/deploy/resources/cluster_role.yaml
@@ -33,6 +33,26 @@
     - apiGroups: ["addon.open-cluster-management.io"]
       resources: ["managedclusteraddons/status"]
       verbs: ["update", "patch"]
+    - apiGroups: ["addon.open-cluster-management.io"]
+      resources: ["addontemplates", "addondeploymentconfigs"]
+      verbs: ["get", "list", "watch"]
     - apiGroups: ["work.open-cluster-management.io"]
       resources: ["manifestworks"]
-      verbs: ["get", "list", "watch"]
+      verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
+    # addon template controller needs these permissions to approve CSR
+    - apiGroups: ["certificates.k8s.io"]
+      resources: ["certificatesigningrequests"]
+      verbs: ["create", "get", "list", "watch"]
+    - apiGroups: ["certificates.k8s.io"]
+      resources: ["certificatesigningrequests/approval", "certificatesigningrequests/status"]
+      verbs: ["update"]
+    - apiGroups: ["certificates.k8s.io"]
+      resources: ["signers"]
+      verbs: ["approve", "sign"]
+    - apiGroups: ["rbac.authorization.k8s.io"]
+      resources: ["rolebindings"]
+      verbs: ["get", "list", "watch", "create", "delete"]
+    # addon template controller needs these permissions to sign CA
+    - apiGroups: [""]
+      resources: ["secrets"]
+      verbs: ["get"]
diff --git a/examples/deploy/addon/helloworld-template/kustomization.yaml b/examples/deploy/addon/helloworld-template/kustomization.yaml
@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: hub
+
+resources:
+- resources/addon_deployment_config.yaml
+- resources/addon_template.yaml
+- resources/cluster_management_addon.yaml
+- resources/managed_cluster_addon.yaml
+- resources/cluster_role.yaml
+
+
+images:
+- name: quay.io/open-cluster-management/addon-examples
+  newName: quay.io/open-cluster-management/addon-examples
+  newTag: latest
diff --git a/examples/deploy/addon/helloworld-template/resources/addon_deployment_config.yaml b/examples/deploy/addon/helloworld-template/resources/addon_deployment_config.yaml
@@ -0,0 +1,9 @@
+apiVersion: addon.open-cluster-management.io/v1alpha1
+kind: AddOnDeploymentConfig
+metadata:
+  name: hello-template
+  namespace: hub
+spec:
+  customizedVariables:
+  - name: LOG_LEVEL
+    value: "4"
diff --git a/examples/deploy/addon/helloworld-template/resources/addon_template.yaml b/examples/deploy/addon/helloworld-template/resources/addon_template.yaml
@@ -0,0 +1,102 @@
+apiVersion: addon.open-cluster-management.io/v1alpha1
+kind: AddOnTemplate
+metadata:
+  name: hello-template
+spec:
+  addonName: hello-template
+  agentSpec:
+      workload:
+        manifests:
+          - kind: Deployment
+            apiVersion: apps/v1
+            metadata:
+              name: hello-template-agent
+              namespace: open-cluster-management-agent-addon
+              annotations:
+                "addon.open-cluster-management.io/deletion-orphan": ""
+              labels:
+                app: hello-template-agent
+            spec:
+              replicas: 1
+              selector:
+                matchLabels:
+                  app: hello-template-agent
+              template:
+                metadata:
+                  labels:
+                    app: hello-template-agent
+                spec:
+                  serviceAccountName: hello-template-agent-sa
+                  containers:
+                    - name: helloworld-agent
+                      image: quay.io/open-cluster-management/addon-examples
+                      imagePullPolicy: IfNotPresent
+                      args:
+                        - "/helloworld_helm"
+                        - "agent"
+                        - "--cluster-name={{CLUSTER_NAME}}"
+                        - "--addon-namespace=open-cluster-management-agent-addon"
+                        - "--addon-name=hello-template"
+                        - "--hub-kubeconfig={{HUB_KUBECONFIG}}"
+                      env:
+                        - name: LOG_LEVEL
+                          value: "{{LOG_LEVEL}}" # addonDeploymentConfig variables
+          - kind: ServiceAccount
+            apiVersion: v1
+            metadata:
+              name: hello-template-agent-sa
+              namespace: open-cluster-management-agent-addon
+              annotations:
+                "addon.open-cluster-management.io/deletion-orphan": ""
+          - kind: ClusterRoleBinding
+            apiVersion: rbac.authorization.k8s.io/v1
+            metadata:
+              name: hello-template-agent
+            roleRef:
+              apiGroup: rbac.authorization.k8s.io
+              kind: ClusterRole
+              name: cluster-admin
+            subjects:
+              - kind: ServiceAccount
+                name: hello-template-agent-sa
+                namespace: open-cluster-management-agent-addon
+          - kind: Job
+            apiVersion: batch/v1
+            metadata:
+              name: hello-template-cleanup-configmap
+              namespace: open-cluster-management-agent-addon
+              annotations:
+                "addon.open-cluster-management.io/addon-pre-delete": ""
+            spec:
+              manualSelector: true
+              selector:
+                matchLabels:
+                  job: hello-template-cleanup-configmap
+              template:
+                metadata:
+                  labels:
+                    job: hello-template-cleanup-configmap
+                spec:
+                  serviceAccountName: hello-template-agent-sa
+                  restartPolicy: Never
+                  containers:
+                  - name: hello-template-agent
+                    image: quay.io/open-cluster-management/addon-examples
+                    imagePullPolicy: IfNotPresent
+                    args:
+                      - "/helloworld_helm"
+                      - "cleanup"
+                      - "--addon-namespace=open-cluster-management-agent-addon"
+  registration:
+    # kubeClient or custom signer, if kubeClient, user and group is in a certain format.
+    # user is "system:open-cluster-management:cluster:{clusterName}:addon:{addonName}:agent:{agentName}"
+    # group is ["system:open-cluster-management:cluster:{clusterName}:addon:{addonName}",
+    #           "system:open-cluster-management:addon:{addonName}", "system:authenticated"]
+    - type: KubeClient
+      kubeClient:
+        hubPermissions:
+          - type: CurrentCluster
+            roleRef:
+              apiGroup: rbac.authorization.k8s.io
+              kind: ClusterRole
+              name: cm-admin
diff --git a/examples/deploy/addon/helloworld-template/resources/cluster_management_addon.yaml b/examples/deploy/addon/helloworld-template/resources/cluster_management_addon.yaml
@@ -0,0 +1,20 @@
+apiVersion: addon.open-cluster-management.io/v1alpha1
+kind: ClusterManagementAddOn
+metadata:
+  name: hello-template
+  annotations:
+    addon.open-cluster-management.io/lifecycle: "addon-manager"
+spec:
+  addOnMeta:
+    description: hello-template
+    displayName: hello-template
+  supportedConfigs:
+  - group: addon.open-cluster-management.io
+    resource: addontemplates
+    defaultConfig:
+      name: hello-template
+  - group: addon.open-cluster-management.io
+    resource: addondeploymentconfigs
+    defaultConfig:
+      name: hello-template
+      namespace: hub
diff --git a/examples/deploy/addon/helloworld-template/resources/cluster_role.yaml b/examples/deploy/addon/helloworld-template/resources/cluster_role.yaml
@@ -0,0 +1,25 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: cm-admin
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+      - delete
+  - apiGroups:
+      - "addon.open-cluster-management.io"
+    resources:
+      - managedclusteraddons
+    verbs:
+      - get
+      - list
+      - watch
diff --git a/examples/deploy/addon/helloworld-template/resources/managed_cluster_addon.yaml b/examples/deploy/addon/helloworld-template/resources/managed_cluster_addon.yaml
@@ -0,0 +1,7 @@
+apiVersion: addon.open-cluster-management.io/v1alpha1
+kind: ManagedClusterAddOn
+metadata:
+  name: hello-template
+  namespace: hub
+spec:
+  installNamespace: open-cluster-management-agent-addon
diff --git a/examples/deploy/hosted-ocm/install.sh b/examples/deploy/hosted-ocm/install.sh
@@ -77,13 +77,14 @@ ${KUBECTL} --context="${cluster_context}" -n kube-system scale --replicas=1 depl
 echo "###### loading image: ${EXAMPLE_IMAGE_NAME}"
 ${KIND} load docker-image ${EXAMPLE_IMAGE_NAME} --name ${MANAGED_CLUSTER_NAME}
 
-echo "###### deploy registration-operator"
-rm -rf "$WORK_DIR/registration-operator"
-git clone https://github.com/open-cluster-management-io/registration-operator.git "$WORK_DIR/registration-operator"
-${KUBECTL} apply -k "$WORK_DIR/registration-operator/deploy/cluster-manager/config/manifests"
-${KUBECTL} apply -k "$WORK_DIR/registration-operator/deploy/cluster-manager/config/samples"
-${KUBECTL} apply -k "$WORK_DIR/registration-operator/deploy/klusterlet/config/manifests"
-rm -rf "$WORK_DIR/registration-operator"
+echo "###### deploy operators"
+rm -rf "$WORK_DIR/_repo_ocm"
+git clone --depth 1 --branch main https://github.com/open-cluster-management-io/ocm.git "$WORK_DIR/_repo_ocm"
+
+${KUBECTL} apply -k "$WORK_DIR/_repo_ocm/deploy/cluster-manager/config/manifests"
+${KUBECTL} apply -k "$WORK_DIR/_repo_ocm/deploy/cluster-manager/config/samples"
+${KUBECTL} apply -k "$WORK_DIR/_repo_ocm/deploy/klusterlet/config/manifests"
+rm -rf "$WORK_DIR/_repo_ocm"
 
 ${KUBECTL} get ns open-cluster-management-agent || ${KUBECTL} create ns open-cluster-management-agent
 

diff --git a/examples/deploy/ocm/install.sh b/examples/deploy/ocm/install.sh
@@ -5,18 +5,18 @@ set -o pipefail
 
 KUBECTL=${KUBECTL:-kubectl}
 
-rm -rf registration-operator
+rm -rf _repo_ocm
 
-echo "############  Cloning registration-operator"
-git clone https://github.com/open-cluster-management-io/registration-operator.git
+echo "############  Cloning ocm repo"
+git clone --depth 1 --branch main https://github.com/open-cluster-management-io/ocm.git _repo_ocm
 
-cd registration-operator || {
-  printf "cd failed, registration-operator does not exist"
+cd _repo_ocm || {
+  printf "cd failed, _repo_ocm does not exist"
   return 1
 }
 
-echo "############  Deploying"
-make deploy
+echo "############  Deploying operators"
+make deploy-hub cluster-ip deploy-spoke-operator apply-spoke-cr
 if [ $? -ne 0 ]; then
  echo "############  Failed to deploy"
  exit 1
@@ -72,6 +72,6 @@ echo "############  All-in-one env is installed successfully!!"
 
 echo "############  Cleanup"
 cd ../ || exist
-rm -rf registration-operator
+rm -rf _repo_ocm
 
 echo "############  Finished installation!!!"
diff --git a/examples/helloworld/helloworld_test.go b/examples/helloworld/helloworld_test.go
@@ -10,11 +10,13 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	utilrand "k8s.io/apimachinery/pkg/util/rand"
 	"k8s.io/klog/v2"
-	"open-cluster-management.io/addon-framework/pkg/addonfactory"
-	"open-cluster-management.io/addon-framework/pkg/addonmanager/addontesting"
 	addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
 	fakeaddon "open-cluster-management.io/api/client/addon/clientset/versioned/fake"
 	clusterv1 "open-cluster-management.io/api/cluster/v1"
+
+	"open-cluster-management.io/addon-framework/pkg/addonfactory"
+	"open-cluster-management.io/addon-framework/pkg/addonmanager/addontesting"
+	"open-cluster-management.io/addon-framework/pkg/utils"
 )
 
 var (
@@ -149,7 +151,7 @@ func TestManifestAddonAgent(t *testing.T) {
 		fakeAddonClient := fakeaddon.NewSimpleClientset(c.configs...)
 
 		agentAddon, err := addonfactory.NewAgentAddonFactory(AddonName, FS, "manifests/templates").
-			WithConfigGVRs(addonfactory.AddOnDeploymentConfigGVR).
+			WithConfigGVRs(utils.AddOnDeploymentConfigGVR).
 			WithGetValuesFuncs(
 				GetDefaultValues,
 				addonfactory.GetAddOnDeploymentConfigValues(

diff --git a/examples/helloworld_agent/agent.go b/examples/helloworld_agent/agent.go
@@ -60,6 +60,7 @@ func (o *AgentOptions) AddFlags(cmd *cobra.Command) {
 		"Location of kubeconfig file to connect to the managed cluster.")
 	flags.StringVar(&o.SpokeClusterName, "cluster-name", o.SpokeClusterName, "Name of spoke cluster.")
 	flags.StringVar(&o.AddonNamespace, "addon-namespace", o.AddonNamespace, "Installation namespace of addon.")
+	flags.StringVar(&o.AddonName, "addon-name", o.AddonName, "name of the addon.")
 }
 
 // RunAgent starts the controllers on agent to process work from hub.

diff --git a/examples/helloworld_agent/cleanup_agent.go b/examples/helloworld_agent/cleanup_agent.go
@@ -27,13 +27,12 @@ func NewCleanupAgentCommand(addonName string) *cobra.Command {
 
 // CleanupAgentOptions defines the flags for workload agent
 type CleanupAgentOptions struct {
-	AddonName             string
 	AddonNamespace        string
 	ManagedKubeconfigFile string
 }
 
 func NewCleanupAgentOptions(addonName string) *CleanupAgentOptions {
-	return &CleanupAgentOptions{AddonName: addonName}
+	return &CleanupAgentOptions{}
 }
 
 func (o *CleanupAgentOptions) AddFlags(cmd *cobra.Command) {