diff --git a/charts/karmada/templates/_helpers.tpl b/charts/karmada/templates/_helpers.tpl index 78abfd86921d..3753e814cfd4 100644 --- a/charts/karmada/templates/_helpers.tpl +++ b/charts/karmada/templates/_helpers.tpl @@ -306,6 +306,10 @@ app: {{- include "karmada.name" .}}-search {{- include "karmada.commonLabels" . -}} {{- end -}} +{{- define "karmada.staticResourceJob.labels" -}} +{{- include "karmada.commonLabels" . -}} +{{- end -}} + {{- define "karmada.postInstallJob.labels" -}} {{- include "karmada.commonLabels" . -}} {{- end -}} @@ -574,3 +578,55 @@ Return the proper Docker Image Registry Secret Names {{- end }} {{- end }} {{- end -}} + +{{- define "karmada.init-sa-secret.volume" -}} +{{- $name := include "karmada.name" . -}} +- name: init-sa-secret + secret: + secretName: {{ $name }}-hook-job +{{- end -}} + +{{- define "karmada.init-sa-secret.volumeMount" -}} +- name: init-sa-secret + mountPath: /opt/mount +{{- end -}} + +{{- define "karmada.initContainer.build-kubeconfig" -}} +TOKEN=$(cat /opt/mount/token) +kubectl config set-cluster karmada-host --server=https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT} --certificate-authority=/opt/mount/ca.crt +kubectl config set-credentials default --token=$TOKEN +kubectl config set-context karmada-host-context --cluster=karmada-host --user=default --namespace=default +kubectl config use-context karmada-host-context +{{- end -}} + +{{- define "karmada.initContainer.waitEtcd" -}} +- name: wait + image: {{ include "karmada.kubectl.image" . }} + imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + bash <<'EOF' + {{- include "karmada.initContainer.build-kubeconfig" . | nindent 6 }} + kubectl rollout status statefulset etcd -n {{ include "karmada.namespace" . }} + EOF + volumeMounts: + {{- include "karmada.init-sa-secret.volumeMount" .| nindent 4 }} +{{- end -}} + +{{- define "karmada.initContainer.waitStaticResource" -}} +- name: wait + image: {{ include "karmada.kubectl.image" . }} + imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + bash <<'EOF' + {{- include "karmada.initContainer.build-kubeconfig" . | nindent 6 }} + kubectl wait --for=condition=complete job {{ include "karmada.name" . }}-static-resource -n {{ include "karmada.namespace" . }} + EOF + volumeMounts: + {{- include "karmada.init-sa-secret.volumeMount" .| nindent 4 }} +{{- end -}} diff --git a/charts/karmada/templates/karmada-aggregated-apiserver.yaml b/charts/karmada/templates/karmada-aggregated-apiserver.yaml index 3d572c5f4f74..cc5e45fdaa35 100644 --- a/charts/karmada/templates/karmada-aggregated-apiserver.yaml +++ b/charts/karmada/templates/karmada-aggregated-apiserver.yaml @@ -29,6 +29,8 @@ spec: spec: {{- include "karmada.aggregatedApiServer.imagePullSecrets" . | nindent 6 }} automountServiceAccountToken: false + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-aggregated-apiserver image: {{ template "karmada.aggregatedApiServer.image" . }} @@ -96,6 +98,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} - name: apiserver-cert secret: diff --git a/charts/karmada/templates/karmada-apiserver.yaml b/charts/karmada/templates/karmada-apiserver.yaml index a788d2d2be2b..4c561ec240fd 100644 --- a/charts/karmada/templates/karmada-apiserver.yaml +++ b/charts/karmada/templates/karmada-apiserver.yaml @@ -28,6 +28,8 @@ spec: spec: {{- include "karmada.apiServer.imagePullSecrets" . | nindent 6 }} automountServiceAccountToken: false + initContainers: + {{- include "karmada.initContainer.waitEtcd" . | nindent 8 }} containers: - name: {{ $name }}-apiserver image: {{ template "karmada.apiServer.image" . }} @@ -135,6 +137,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} - name: apiserver-cert secret: secretName: {{ $name }}-cert diff --git a/charts/karmada/templates/karmada-controller-manager.yaml b/charts/karmada/templates/karmada-controller-manager.yaml index 2a4e565f3d49..3415573ac66b 100644 --- a/charts/karmada/templates/karmada-controller-manager.yaml +++ b/charts/karmada/templates/karmada-controller-manager.yaml @@ -42,7 +42,10 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-controller-manager image: {{ template "karmada.controllerManager.image" . }} diff --git a/charts/karmada/templates/karmada-descheduler.yaml b/charts/karmada/templates/karmada-descheduler.yaml index caf9b8d29757..c55109244aa3 100644 --- a/charts/karmada/templates/karmada-descheduler.yaml +++ b/charts/karmada/templates/karmada-descheduler.yaml @@ -41,6 +41,8 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} automountServiceAccountToken: false + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-descheduler image: {{ template "karmada.descheduler.image" . }} @@ -65,6 +67,7 @@ spec: resources: {{- toYaml .Values.descheduler.resources | nindent 12 }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.descheduler.kubeconfig.volume" . | nindent 8 }} {{ if .Values.descheduler.podDisruptionBudget }} diff --git a/charts/karmada/templates/karmada-metrics-adapter.yaml b/charts/karmada/templates/karmada-metrics-adapter.yaml index 49f8fe29e0ec..16bc12bb5dd7 100644 --- a/charts/karmada/templates/karmada-metrics-adapter.yaml +++ b/charts/karmada/templates/karmada-metrics-adapter.yaml @@ -30,6 +30,8 @@ spec: spec: {{- include "karmada.metricsAdapter.imagePullSecrets" . | nindent 6 }} automountServiceAccountToken: false + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-aggregated-apiserver image: {{ template "karmada.metricsAdapter.image" . }} @@ -81,6 +83,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} - name: apiserver-cert secret: diff --git a/charts/karmada/templates/karmada-scheduler.yaml b/charts/karmada/templates/karmada-scheduler.yaml index 503c050b16b2..a6933df792f4 100644 --- a/charts/karmada/templates/karmada-scheduler.yaml +++ b/charts/karmada/templates/karmada-scheduler.yaml @@ -41,6 +41,8 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} automountServiceAccountToken: false + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-scheduler image: {{ template "karmada.scheduler.image" .}} @@ -65,6 +67,7 @@ spec: resources: {{- toYaml .Values.scheduler.resources | nindent 12 }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} {{ if .Values.scheduler.podDisruptionBudget }} diff --git a/charts/karmada/templates/karmada-search.yaml b/charts/karmada/templates/karmada-search.yaml index 5478b46cb486..bb4c91fa9824 100644 --- a/charts/karmada/templates/karmada-search.yaml +++ b/charts/karmada/templates/karmada-search.yaml @@ -40,6 +40,8 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} automountServiceAccountToken: false + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-search image: {{ template "karmada.search.image" . }} @@ -90,6 +92,7 @@ spec: resources: {{- toYaml .Values.apiServer.resources | nindent 12 }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.search.kubeconfig.volume" . | nindent 8 }} {{- include "karmada.search.etcd.cert.volume" . | nindent 8 }} --- diff --git a/charts/karmada/templates/karmada-static-resource-job.yaml b/charts/karmada/templates/karmada-static-resource-job.yaml new file mode 100644 index 000000000000..1d2ea6179910 --- /dev/null +++ b/charts/karmada/templates/karmada-static-resource-job.yaml @@ -0,0 +1,105 @@ +{{- $name := include "karmada.name" . -}} +{{- $namespace := include "karmada.namespace" . -}} +{{- if eq .Values.installMode "host" }} +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{ $name }}-static-resource" + namespace: {{ $namespace }} + labels: + {{- include "karmada.staticResourceJob.labels" . | nindent 4 }} +spec: + parallelism: 1 + completions: 1 + template: + metadata: + name: {{ $name }} + labels: + {{- include "karmada.staticResourceJob.labels" . | nindent 8 }} + spec: + {{- include "karmada.imagePullSecrets" . | nindent 6 }} + {{- with .Values.staticResourceJob.tolerations}} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.staticResourceJob.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ $name }}-hook-job + restartPolicy: Never + containers: + - name: post-install + image: {{ template "karmada.kubectl.image" . }} + imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }} + command: + - /bin/sh + - -c + - | + bash <<'EOF' + set -ex + kubectl rollout status deployment {{ $name }}-apiserver -n {{ $namespace }} + kubectl apply -k /crds --kubeconfig /etc/kubeconfig + kubectl apply -f /static-resources/system-namespace.yaml --kubeconfig /etc/kubeconfig + kubectl apply -f /static-resources/ --kubeconfig /etc/kubeconfig + EOF + volumeMounts: + - name: {{ $name }}-crds-kustomization + mountPath: /crds + - name: {{ $name }}-crds-patches + mountPath: /crds/patches + - name: {{ $name }}-crds-autoscaling-bases + mountPath: /crds/bases/autoscaling + - name: {{ $name }}-crds-config-bases + mountPath: /crds/bases/config + - name: {{ $name }}-crds-multicluster-bases + mountPath: /crds/bases/multicluster + - name: {{ $name }}-crds-networking-bases + mountPath: /crds/bases/networking + - name: {{ $name }}-crds-policy-bases + mountPath: /crds/bases/policy + - name: {{ $name }}-crds-remedy-bases + mountPath: /crds/bases/remedy + - name: {{ $name }}-crds-work-bases + mountPath: /crds/bases/work + - name: {{ $name }}-crds-apps-bases + mountPath: /crds/bases/apps + - name: {{ $name }}-static-resources + mountPath: /static-resources + {{ include "karmada.kubeconfig.volumeMount" . | nindent 10 }} + volumes: + - name: {{ $name }}-crds-kustomization + configMap: + name: {{ $name }}-crds-kustomization + - name: {{ $name }}-crds-patches + configMap: + name: {{ $name }}-crds-patches + - name: {{ $name }}-crds-autoscaling-bases + configMap: + name: {{ $name }}-crds-autoscaling-bases + - name: {{ $name }}-crds-config-bases + configMap: + name: {{ $name }}-crds-config-bases + - name: {{ $name }}-crds-multicluster-bases + configMap: + name: {{ $name }}-crds-multicluster-bases + - name: {{ $name }}-crds-networking-bases + configMap: + name: {{ $name }}-crds-networking-bases + - name: {{ $name }}-crds-policy-bases + configMap: + name: {{ $name }}-crds-policy-bases + - name: {{ $name }}-crds-remedy-bases + configMap: + name: {{ $name }}-crds-remedy-bases + - name: {{ $name }}-crds-work-bases + configMap: + name: {{ $name }}-crds-work-bases + - name: {{ $name }}-crds-apps-bases + configMap: + name: {{ $name }}-crds-apps-bases + - name: {{ $name }}-static-resources + configMap: + name: {{ $name }}-static-resources + {{ include "karmada.kubeconfig.volume" . | nindent 8 }} +{{- end }} diff --git a/charts/karmada/templates/karmada-webhook.yaml b/charts/karmada/templates/karmada-webhook.yaml index 60bd70391425..3f20dfe71406 100644 --- a/charts/karmada/templates/karmada-webhook.yaml +++ b/charts/karmada/templates/karmada-webhook.yaml @@ -41,6 +41,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - name: {{ $name }}-webhook image: {{ template "karmada.webhook.image" . }} @@ -66,6 +68,7 @@ spec: resources: {{- toYaml .Values.webhook.resources | nindent 12 }} volumes: + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} - name: {{ $name }}-webhook-cert-secret secret: diff --git a/charts/karmada/templates/kube-controller-manager.yaml b/charts/karmada/templates/kube-controller-manager.yaml index 2e966253d18b..ab9f40223c7f 100644 --- a/charts/karmada/templates/kube-controller-manager.yaml +++ b/charts/karmada/templates/kube-controller-manager.yaml @@ -41,6 +41,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} + initContainers: + {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }} containers: - command: - kube-controller-manager @@ -87,6 +89,7 @@ spec: - name: apisever-cert secret: secretName: {{ $name }}-cert + {{- include "karmada.init-sa-secret.volume" . | nindent 8 }} {{- include "karmada.kubeconfig.volume" . | nindent 8 }} {{ if .Values.kubeControllerManager.podDisruptionBudget }} diff --git a/charts/karmada/templates/post-delete-job.yaml b/charts/karmada/templates/post-delete-job.yaml index 699c38ac66b2..0b9907873daf 100644 --- a/charts/karmada/templates/post-delete-job.yaml +++ b/charts/karmada/templates/post-delete-job.yaml @@ -33,7 +33,7 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ $name }}-pre-job + serviceAccountName: {{ $name }}-hook-job restartPolicy: Never containers: - name: post-delete @@ -47,14 +47,13 @@ spec: set -ex kubectl delete -f /opt/mount/ --ignore-not-found=true kubectl delete -f /opt/crds/ --ignore-not-found=true -R + kubectl delete -f /opt/static-resources/ --ignore-not-found=true -R kubectl delete cm/{{ $name }}-config -n {{ $namespace }} --ignore-not-found=true kubectl delete deployment/{{ $name }}-controller-manager -n {{ $namespace }} --ignore-not-found=true EOF volumeMounts: - name: mount mountPath: /opt/mount - - name: crds - mountPath: /opt/crds - name: crds-autoscaling-base mountPath: /opt/crds/base/autoscaling - name: crds-config-base @@ -69,35 +68,37 @@ spec: mountPath: /opt/crds/base/remedy - name: crds-work-base mountPath: /opt/crds/base/work + - name: static-resources + mountPath: /opt/static-resources volumes: - name: mount configMap: name: {{ $name }}-config - - name: crds - configMap: - name: {{ $name }}-crds-config - name: crds-autoscaling-base configMap: - name: {{ $name }}-crds-autoscaling-bases-config + name: {{ $name }}-crds-autoscaling-bases - name: crds-config-base configMap: - name: {{ $name }}-crds-config-bases-config + name: {{ $name }}-crds-config-bases - name: crds-multicluster-base configMap: - name: {{ $name }}-crds-multicluster-bases-config + name: {{ $name }}-crds-multicluster-bases - name: crds-networking-base configMap: - name: {{ $name }}-crds-networking-bases-config + name: {{ $name }}-crds-networking-bases - name: crds-policy-base configMap: - name: {{ $name }}-crds-policy-bases-config + name: {{ $name }}-crds-policy-bases - name: crds-remedy-base configMap: - name: {{ $name }}-crds-remedy-bases-config + name: {{ $name }}-crds-remedy-bases - name: crds-work-base configMap: - name: {{ $name }}-crds-work-bases-config + name: {{ $name }}-crds-work-bases - name: crds-apps-base configMap: - name: {{ $name }}-crds-apps-bases-config + name: {{ $name }}-crds-apps-bases + - name: static-resources + configMap: + name: {{ $name }}-static-resources {{- end }} diff --git a/charts/karmada/templates/post-install-job.yaml b/charts/karmada/templates/post-install-job.yaml index c14a43079fa7..0ed011525311 100644 --- a/charts/karmada/templates/post-install-job.yaml +++ b/charts/karmada/templates/post-install-job.yaml @@ -38,6 +38,7 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} + serviceAccountName: {{ $name }}-hook-job restartPolicy: Never containers: - name: post-install @@ -49,66 +50,7 @@ spec: - | bash <<'EOF' set -ex - kubectl apply -k /crds --kubeconfig /etc/kubeconfig - kubectl apply -f /static-resources --kubeconfig /etc/kubeconfig + kubectl delete job {{ $name }}-static-resource -n {{ $namespace }} + kubectl delete secret {{ $name }}-hook-job -n {{ $namespace }} EOF - volumeMounts: - - name: {{ $name }}-crds-kustomization - mountPath: /crds - - name: {{ $name }}-crds-patches - mountPath: /crds/patches - - name: {{ $name }}-crds-autoscaling-bases - mountPath: /crds/bases/autoscaling - - name: {{ $name }}-crds-config-bases - mountPath: /crds/bases/config - - name: {{ $name }}-crds-multicluster-bases - mountPath: /crds/bases/multicluster - - name: {{ $name }}-crds-networking-bases - mountPath: /crds/bases/networking - - name: {{ $name }}-crds-policy-bases - mountPath: /crds/bases/policy - - name: {{ $name }}-crds-remedy-bases - mountPath: /crds/bases/remedy - - name: {{ $name }}-crds-work-bases - mountPath: /crds/bases/work - - name: {{ $name }}-crds-apps-bases - mountPath: /crds/bases/apps - - name: {{ $name }}-static-resources - mountPath: /static-resources - {{ include "karmada.kubeconfig.volumeMount" . | nindent 10 }} - volumes: - - name: {{ $name }}-crds-kustomization - configMap: - name: {{ $name }}-crds-kustomization - - name: {{ $name }}-crds-patches - configMap: - name: {{ $name }}-crds-patches - - name: {{ $name }}-crds-autoscaling-bases - configMap: - name: {{ $name }}-crds-autoscaling-bases - - name: {{ $name }}-crds-config-bases - configMap: - name: {{ $name }}-crds-config-bases - - name: {{ $name }}-crds-multicluster-bases - configMap: - name: {{ $name }}-crds-multicluster-bases - - name: {{ $name }}-crds-networking-bases - configMap: - name: {{ $name }}-crds-networking-bases - - name: {{ $name }}-crds-policy-bases - configMap: - name: {{ $name }}-crds-policy-bases - - name: {{ $name }}-crds-remedy-bases - configMap: - name: {{ $name }}-crds-remedy-bases - - name: {{ $name }}-crds-work-bases - configMap: - name: {{ $name }}-crds-work-bases - - name: {{ $name }}-crds-apps-bases - configMap: - name: {{ $name }}-crds-apps-bases - - name: {{ $name }}-static-resources - configMap: - name: {{ $name }}-static-resources - {{ include "karmada.kubeconfig.volume" . | nindent 8 }} {{- end }} diff --git a/charts/karmada/templates/pre-install-job.yaml b/charts/karmada/templates/pre-install-job.yaml index c5eca2551cce..cc8fe785f67a 100644 --- a/charts/karmada/templates/pre-install-job.yaml +++ b/charts/karmada/templates/pre-install-job.yaml @@ -372,7 +372,7 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - serviceAccountName: {{ $name }}-pre-job + serviceAccountName: {{ $name }}-hook-job restartPolicy: Never initContainers: - name: init @@ -445,12 +445,11 @@ spec: name: {{ $name }}-config - name: configs emptyDir: {} - --- apiVersion: v1 kind: ServiceAccount metadata: - name: {{ $name }}-pre-job + name: {{ $name }}-hook-job namespace: {{ $namespace }} annotations: "helm.sh/hook": pre-install @@ -460,10 +459,25 @@ metadata: {{- include "karmada.preInstallJob.labels" . | nindent 4 }} {{- end }} --- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }}-hook-job + namespace: {{ $namespace }} + annotations: + "kubernetes.io/service-account.name": {{ $name }}-hook-job + "helm.sh/hook": pre-install + "helm.sh/hook-weight": "1" + {{- if "karmada.preInstallJob.labels" }} + labels: + {{- include "karmada.preInstallJob.labels" . | nindent 4 }} + {{- end }} +type: kubernetes.io/service-account-token +--- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - name: {{ $name }}-pre-job + name: {{ $name }}-hook-job annotations: "helm.sh/hook": pre-install "helm.sh/hook-weight": "1" @@ -481,7 +495,7 @@ rules: apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: {{ $name }}-pre-job + name: {{ $name }}-hook-job annotations: "helm.sh/hook": pre-install "helm.sh/hook-weight": "1" @@ -492,10 +506,10 @@ metadata: roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: {{ $name }}-pre-job + name: {{ $name }}-hook-job subjects: - kind: ServiceAccount - name: {{ $name }}-pre-job + name: {{ $name }}-hook-job namespace: {{ $namespace }} --- {{- end }} diff --git a/charts/karmada/values.yaml b/charts/karmada/values.yaml index c120df909fd8..114f02b9a7a2 100644 --- a/charts/karmada/values.yaml +++ b/charts/karmada/values.yaml @@ -98,6 +98,11 @@ preInstallJob: ## Define policies that determine when to delete corresponding hook resources: before-hook-creation,hook-succeeded,hook-failed hookDeletePolicy: "hook-succeeded" +## static-resource job config +staticResourceJob: + tolerations: [] + nodeSelector: {} + ## post-install job config postInstallJob: tolerations: []