From cd0725fc0e5ae7c9df33c1c69d8931fb4414dc61 Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Tue, 16 May 2023 12:34:29 +0300 Subject: [PATCH 1/2] Add CRDs to the network-operator image Signed-off-by: Yury Kulazhenkov --- Dockerfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Dockerfile b/Dockerfile index be7080a7..cbc504ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,8 @@ # Build the manager binary FROM golang:1.20 as builder +ARG TARGETARCH=amd64 + WORKDIR /workspace # Copy the Go Modules manifests COPY go.mod go.mod @@ -29,6 +31,17 @@ COPY api/ api/ COPY controllers/ controllers/ COPY pkg/ pkg/ +# Add kubectl tool +RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${TARGETARCH}/kubectl" +RUN chmod +x ./kubectl + +# Add helm chart DIR to get CRDs from it +COPY deployment/network-operator chart +# Update helm dependencies and copy CRDs from them +RUN mkdir crds && \ + cp -r chart/crds /workspace/crds/network-operator/ && \ + cp -r chart/charts/sriov-network-operator/crds /workspace/crds/sriov-network-operator/ + # Build RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go @@ -57,6 +70,9 @@ LABEL org.label-schema.vcs-url="https://github.com/Mellanox/network-operator" WORKDIR / COPY --from=builder /workspace/manager . +COPY --from=builder /workspace/kubectl /usr/local/bin +COPY --from=builder /workspace/crds /crds + COPY manifests/ manifests/ USER 65532:65532 From 43ace75746e66f492cf55db1fe914e8aca0dd46b Mon Sep 17 00:00:00 2001 From: Yury Kulazhenkov Date: Tue, 16 May 2023 13:52:52 +0300 Subject: [PATCH 2/2] Add helm pre-upgrade hook for auto CRD upgrade Signed-off-by: Yury Kulazhenkov --- Dockerfile | 2 +- deployment/network-operator/README.md | 41 ++++++++++--------- .../templates/upgrade-crd.yaml | 37 +++++++++++++++++ deployment/network-operator/values.yaml | 2 + 4 files changed, 61 insertions(+), 21 deletions(-) create mode 100644 deployment/network-operator/templates/upgrade-crd.yaml diff --git a/Dockerfile b/Dockerfile index cbc504ad..ffe96743 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ RUN chmod +x ./kubectl # Add helm chart DIR to get CRDs from it COPY deployment/network-operator chart -# Update helm dependencies and copy CRDs from them +# copy CRDs from helm charts RUN mkdir crds && \ cp -r chart/crds /workspace/crds/network-operator/ && \ cp -r chart/charts/sriov-network-operator/crds /workspace/crds/sriov-network-operator/ diff --git a/deployment/network-operator/README.md b/deployment/network-operator/README.md index af8c5e67..b2de97dc 100644 --- a/deployment/network-operator/README.md +++ b/deployment/network-operator/README.md @@ -169,11 +169,6 @@ Notes: Before starting the upgrade to a specific release version, please, check release notes for this version to ensure that no additional actions are required. -Since Helm doesn’t support auto-upgrade of existing CRDs, the user needs to follow a two-step process to upgrade the -network-operator release: - -- Upgrade CRD to the latest version -- Apply helm chart update ### Check available releases @@ -183,7 +178,12 @@ helm search repo mellanox/network-operator -l > __NOTE__: add `--devel` option if you want to list beta releases as well -### Download CRDs for the specific release +### Upgrade CRDs to compatible version + +The network-operator helm chart contains a pre-upgrade hook that will automatically upgrade required CRDs in the cluster. +The hook is enabled by default. If you don't want to upgrade CRDs with helm automatically, +you can disable auto upgrade by setting `upgradeCRDs: false` in the helm chart values. +Then you can follow the guide below to download and apply CRDs for the concrete version of the network-operator. It is possible to retrieve updated CRDs from the Helm chart or from the release branch on GitHub. Example bellow show how to download and unpack Helm chart for specified release and then apply CRDs update from it. @@ -331,20 +331,21 @@ parameters. ### General parameters -| Name | Type | Default | description | -|------------------------------------------------------|--------| ------- |-------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `nfd.enabled` | bool | `True` | deploy Node Feature Discovery | -| `sriovNetworkOperator.enabled` | bool | `False` | deploy SR-IOV Network Operator | -| `sriovNetworkOperator.configDaemonNodeSelectorExtra` | object | `{"node-role.kubernetes.io/worker": ""}` | Additional nodeSelector for sriov-network-operator config daemon. These values will be added in addition to default values managed by the network-operator. | -| `psp.enabled` | bool | `False` | deploy Pod Security Policy | -| `imagePullSecrets` | list | `[]` | An optional list of references to secrets to use for pulling any of the Network Operator image if it's not overrided | -| `operator.repository` | string | `nvcr.io/nvidia/cloud-native` | Network Operator image repository | -| `operator.image` | string | `network-operator` | Network Operator image name | -| `operator.tag` | string | `None` | Network Operator image tag, if `None`, then the Chart's `appVersion` will be used | -| `operator.imagePullSecrets` | list | `[]` | An optional list of references to secrets to use for pulling Network Operator image | -| `deployCR` | bool | `false` | Deploy `NicClusterPolicy` custom resource according to provided parameters | -| `nodeAffinity` | yaml | `` | Override the node affinity for various Daemonsets deployed by network operator, e.g. whereabouts, multus, cni-plugins. | -| `tolerations` | yaml | `` | Set additional tolerations for various Daemonsets deployed by network operator, e.g. whereabouts, multus, cni-plugins. | +| Name | Type | Default | description | +|------------------------------------------------------|--------|------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `nfd.enabled` | bool | `True` | deploy Node Feature Discovery | +| `sriovNetworkOperator.enabled` | bool | `False` | deploy SR-IOV Network Operator | +| `upgradeCRDs` | bool | `True` | enable CRDs upgrade with helm pre-upgrade hook | +| `sriovNetworkOperator.configDaemonNodeSelectorExtra` | object | `{"node-role.kubernetes.io/worker": ""}` | Additional nodeSelector for sriov-network-operator config daemon. These values will be added in addition to default values managed by the network-operator. | +| `psp.enabled` | bool | `False` | deploy Pod Security Policy | +| `imagePullSecrets` | list | `[]` | An optional list of references to secrets to use for pulling any of the Network Operator image if it's not overrided | +| `operator.repository` | string | `nvcr.io/nvidia/cloud-native` | Network Operator image repository | +| `operator.image` | string | `network-operator` | Network Operator image name | +| `operator.tag` | string | `None` | Network Operator image tag, if `None`, then the Chart's `appVersion` will be used | +| `operator.imagePullSecrets` | list | `[]` | An optional list of references to secrets to use for pulling Network Operator image | +| `deployCR` | bool | `false` | Deploy `NicClusterPolicy` custom resource according to provided parameters | +| `nodeAffinity` | yaml | `` | Override the node affinity for various Daemonsets deployed by network operator, e.g. whereabouts, multus, cni-plugins. | +| `tolerations` | yaml | `` | Set additional tolerations for various Daemonsets deployed by network operator, e.g. whereabouts, multus, cni-plugins. | #### imagePullSecrets customization diff --git a/deployment/network-operator/templates/upgrade-crd.yaml b/deployment/network-operator/templates/upgrade-crd.yaml new file mode 100644 index 00000000..59fd59f3 --- /dev/null +++ b/deployment/network-operator/templates/upgrade-crd.yaml @@ -0,0 +1,37 @@ +{{- if .Values.upgradeCRDs }} +apiVersion: batch/v1 +kind: Job +metadata: + name: network-operator-upgrade-crd + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + labels: + {{- include "network-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: "network-operator" +spec: + template: + metadata: + name: network-operator-upgrade-crd + labels: + {{- include "network-operator.labels" . | nindent 8 }} + app.kubernetes.io/component: "network-operator" + spec: + serviceAccountName: {{ include "network-operator.fullname" . }} + imagePullSecrets: {{ include "network-operator.operator.imagePullSecrets" . | nindent 6 }} + containers: + - name: upgrade-crd + image: "{{ .Values.operator.repository }}/{{ .Values.operator.image }}:{{ .Values.operator.tag | default .Chart.AppVersion }}" + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + kubectl apply -f /crds/network-operator; + {{- if .Values.sriovNetworkOperator.enabled }} + kubectl apply -f /crds/sriov-network-operator; + {{- end }} + restartPolicy: OnFailure +{{- end }} diff --git a/deployment/network-operator/values.yaml b/deployment/network-operator/values.yaml index 5de84f1f..398953c9 100644 --- a/deployment/network-operator/values.yaml +++ b/deployment/network-operator/values.yaml @@ -22,6 +22,8 @@ nfd: psp: enabled: false +upgradeCRDs: true + sriovNetworkOperator: enabled: false # inject additional values to nodeSelector for config daemon