Add support for imagePullSecrets (#2589)

* Add support for imagePullSecrets in the CDI CR, to support pulling images from repositories that require secrets. The imagePullSecrets is propagated to the following components: cdi-apiserver, cdi-deployment, and cdi-uploadproxy. The definition of imagePullSecrets in cdi-operator must be done manually. Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com> * Modifying code to incorporate review comments. Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com> --------- Signed-off-by: Gleb Aronsky <gleb.aronsky@windriver.com> Co-authored-by: Gleb Aronsky <gleb.aronsky@windriver.com>
kubevirt · Mar 18, 2023 · 4e3ee4c · 4e3ee4c
1 parent 184ee03
commit 4e3ee4c
Show file tree

Hide file tree

Showing 23 changed files with 225 additions and 27 deletions.
diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json
@@ -4044,6 +4044,17 @@
      }
     }
    },
+   "v1.LocalObjectReference": {
+    "description": "LocalObjectReference contains enough information to let you locate the referenced object inside the same namespace.",
+    "type": "object",
+    "properties": {
+     "name": {
+      "description": "Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names",
+      "type": "string"
+     }
+    },
+    "x-kubernetes-map-type": "atomic"
+   },
    "v1.ManagedFieldsEntry": {
     "description": "ManagedFieldsEntry is a workflow-id, a FieldSet and the group version of the resource that the fieldset applies to.",
     "type": "object",
@@ -4893,6 +4904,14 @@
       "description": "FilesystemOverhead describes the space reserved for overhead when using Filesystem volumes. A value is between 0 and 1, if not defined it is 0.055 (5.5% overhead)",
       "$ref": "#/definitions/v1beta1.FilesystemOverhead"
      },
+     "imagePullSecrets": {
+      "description": "The imagePullSecrets used to pull the container images",
+      "type": "array",
+      "items": {
+       "default": {},
+       "$ref": "#/definitions/v1.LocalObjectReference"
+      }
+     },
      "importProxy": {
       "description": "ImportProxy contains importer pod proxy configuration.",
       "$ref": "#/definitions/v1beta1.ImportProxy"
@@ -4939,6 +4958,14 @@
       "description": "FilesystemOverhead describes the space reserved for overhead when using Filesystem volumes. A percentage value is between 0 and 1",
       "$ref": "#/definitions/v1beta1.FilesystemOverhead"
      },
+     "imagePullSecrets": {
+      "description": "The imagePullSecrets used to pull the container images",
+      "type": "array",
+      "items": {
+       "default": {},
+       "$ref": "#/definitions/v1.LocalObjectReference"
+      }
+     },
      "importProxy": {
       "description": "ImportProxy contains importer pod proxy configuration.",
       "$ref": "#/definitions/v1beta1.ImportProxy"

diff --git a/pkg/apis/core/v1beta1/openapi_generated.go b/pkg/apis/core/v1beta1/openapi_generated.go
diff --git a/pkg/controller/clone-controller.go b/pkg/controller/clone-controller.go
@@ -497,6 +497,11 @@ func (r *CloneReconciler) CreateCloneSourcePod(image, pullPolicy string, pvc *co
 		return nil, err
 	}
 
+	imagePullSecrets, err := cc.GetImagePullSecrets(r.client)
+	if err != nil {
+		return nil, err
+	}
+
 	workloadNodePlacement, err := cc.GetWorkloadNodePlacement(r.client)
 	if err != nil {
 		return nil, err
@@ -514,7 +519,7 @@ func (r *CloneReconciler) CreateCloneSourcePod(image, pullPolicy string, pvc *co
 		sourceVolumeMode = corev1.PersistentVolumeFilesystem
 	}
 
-	pod := MakeCloneSourcePodSpec(sourceVolumeMode, image, pullPolicy, sourcePvcName, sourcePvcNamespace, ownerKey, serverCABundle, pvc, podResourceRequirements, workloadNodePlacement)
+	pod := MakeCloneSourcePodSpec(sourceVolumeMode, image, pullPolicy, imagePullSecrets, sourcePvcName, sourcePvcNamespace, ownerKey, serverCABundle, pvc, podResourceRequirements, workloadNodePlacement)
 	util.SetRecommendedLabels(pod, r.installerLabels, "cdi-controller")
 
 	if err := r.client.Create(context.TODO(), pod); err != nil {
@@ -527,7 +532,7 @@ func (r *CloneReconciler) CreateCloneSourcePod(image, pullPolicy string, pvc *co
 }
 
 // MakeCloneSourcePodSpec creates and returns the clone source pod spec based on the target pvc.
-func MakeCloneSourcePodSpec(sourceVolumeMode corev1.PersistentVolumeMode, image, pullPolicy, sourcePvcName, sourcePvcNamespace, ownerRefAnno string,
+func MakeCloneSourcePodSpec(sourceVolumeMode corev1.PersistentVolumeMode, image, pullPolicy string, imagePullSecrets []corev1.LocalObjectReference, sourcePvcName, sourcePvcNamespace, ownerRefAnno string,
 	serverCACert []byte, targetPvc *corev1.PersistentVolumeClaim, resourceRequirements *corev1.ResourceRequirements,
 	workloadNodePlacement *sdkapi.NodePlacement) *corev1.Pod {
 
@@ -617,7 +622,8 @@ func MakeCloneSourcePodSpec(sourceVolumeMode corev1.PersistentVolumeMode, image,
 					},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			ImagePullSecrets: imagePullSecrets,
+			RestartPolicy:    corev1.RestartPolicyOnFailure,
 			Volumes: []corev1.Volume{
 				{
 					Name: cc.DataVolName,

diff --git a/pkg/controller/common/util.go b/pkg/controller/common/util.go
@@ -415,6 +415,17 @@ func GetDefaultPodResourceRequirements(client client.Client) (*v1.ResourceRequir
 	return cdiconfig.Status.DefaultPodResourceRequirements, nil
 }
 
+// GetImagePullSecrets gets the imagePullSecrets needed to pull images from the cdi config
+func GetImagePullSecrets(client client.Client) ([]corev1.LocalObjectReference, error) {
+	cdiconfig := &cdiv1.CDIConfig{}
+	if err := client.Get(context.TODO(), types.NamespacedName{Name: common.ConfigName}, cdiconfig); err != nil {
+		klog.Errorf("Unable to find CDI configuration, %v\n", err)
+		return nil, err
+	}
+
+	return cdiconfig.Status.ImagePullSecrets, nil
+}
+
 // AddVolumeDevices returns VolumeDevice slice with one block device for pods using PV with block volume mode
 func AddVolumeDevices() []v1.VolumeDevice {
 	volumeDevices := []v1.VolumeDevice{

diff --git a/pkg/controller/config-controller.go b/pkg/controller/config-controller.go
@@ -97,6 +97,10 @@ func (r *CDIConfigReconciler) Reconcile(_ context.Context, req reconcile.Request
 		return reconcile.Result{}, err
 	}
 
+	if err := r.reconcileImagePullSecrets(config); err != nil {
+		return reconcile.Result{}, err
+	}
+
 	if err := r.reconcileFilesystemOverhead(config); err != nil {
 		return reconcile.Result{}, err
 	}
@@ -232,6 +236,11 @@ func (r *CDIConfigReconciler) reconcileStorageClass(config *cdiv1.CDIConfig) err
 	return nil
 }
 
+func (r *CDIConfigReconciler) reconcileImagePullSecrets(config *cdiv1.CDIConfig) error {
+	config.Status.ImagePullSecrets = config.Spec.ImagePullSecrets
+	return nil
+}
+
 func (r *CDIConfigReconciler) reconcileDefaultPodResourceRequirements(config *cdiv1.CDIConfig) error {
 	cpuLimit, _ := resource.ParseQuantity(defaultCPULimit)
 	memLimit, _ := resource.ParseQuantity(defaultMemLimit)

diff --git a/pkg/controller/dataimportcron-controller.go b/pkg/controller/dataimportcron-controller.go
@@ -860,6 +860,10 @@ func (r *DataImportCronReconciler) newCronJob(cron *cdiv1.DataImportCron) (*batc
 	addEnvVarFromImportProxyConfig(common.ImportProxyHTTPS)
 	addEnvVarFromImportProxyConfig(common.ImportProxyNoProxy)
 
+	imagePullSecrets, err := cc.GetImagePullSecrets(r.client)
+	if err != nil {
+		return nil, err
+	}
 	cronJobName := GetCronJobName(cron)
 	cronJob := &batchv1.CronJob{
 		ObjectMeta: metav1.ObjectMeta{
@@ -880,6 +884,7 @@ func (r *DataImportCronReconciler) newCronJob(cron *cdiv1.DataImportCron) (*batc
 							Containers:                    []corev1.Container{container},
 							ServiceAccountName:            common.CronJobServiceAccountName,
 							Volumes:                       volumes,
+							ImagePullSecrets:              imagePullSecrets,
 						},
 					},
 					BackoffLimit:            pointer.Int32(2),

diff --git a/pkg/controller/datavolume/clone-controller-base.go b/pkg/controller/datavolume/clone-controller-base.go
@@ -420,6 +420,11 @@ func (r *CloneReconcilerBase) createExpansionPod(pvc *corev1.PersistentVolumeCla
 		return nil, err
 	}
 
+	imagePullSecrets, err := cc.GetImagePullSecrets(r.client)
+	if err != nil {
+		return nil, err
+	}
+
 	workloadNodePlacement, err := cc.GetWorkloadNodePlacement(r.client)
 	if err != nil {
 		return nil, err
@@ -447,7 +452,8 @@ func (r *CloneReconcilerBase) createExpansionPod(pvc *corev1.PersistentVolumeCla
 					Args:            []string{"-c", "echo", "'hello cdi'"},
 				},
 			},
-			RestartPolicy: corev1.RestartPolicyOnFailure,
+			ImagePullSecrets: imagePullSecrets,
+			RestartPolicy:    corev1.RestartPolicyOnFailure,
 			Volumes: []corev1.Volume{
 				{
 					Name: cc.DataVolName,

diff --git a/pkg/controller/datavolume/pvc-clone-controller.go b/pkg/controller/datavolume/pvc-clone-controller.go
@@ -1253,6 +1253,11 @@ func (r *PvcCloneReconciler) makeSizeDetectionPodSpec(
 	if container == nil {
 		return nil
 	}
+	imagePullSecrets, err := cc.GetImagePullSecrets(r.client)
+	if err != nil {
+		return nil
+	}
+
 	// Assemble the pod
 	pod := &corev1.Pod{
 		ObjectMeta: *objectMeta,
@@ -1268,6 +1273,7 @@ func (r *PvcCloneReconciler) makeSizeDetectionPodSpec(
 			Tolerations:       workloadNodePlacement.Tolerations,
 			Affinity:          workloadNodePlacement.Affinity,
 			PriorityClassName: cc.GetPriorityClass(sourcePvc),
+			ImagePullSecrets:  imagePullSecrets,
 		},
 	}
 

diff --git a/pkg/controller/import-controller.go b/pkg/controller/import-controller.go
@@ -111,6 +111,7 @@ type importerPodArgs struct {
 	pvc                     *corev1.PersistentVolumeClaim
 	scratchPvcName          *string
 	podResourceRequirements *corev1.ResourceRequirements
+	imagePullSecrets        []corev1.LocalObjectReference
 	workloadNodePlacement   *sdkapi.NodePlacement
 	vddkImageName           *string
 	priorityClassName       string
@@ -866,6 +867,11 @@ func createImporterPod(log logr.Logger, client client.Client, args *importerPodA
 		return nil, err
 	}
 
+	args.imagePullSecrets, err = cc.GetImagePullSecrets(client)
+	if err != nil {
+		return nil, err
+	}
+
 	args.workloadNodePlacement, err = cc.GetWorkloadNodePlacement(client)
 	if err != nil {
 		return nil, err
@@ -970,6 +976,7 @@ func makeNodeImporterPodSpec(args *importerPodArgs) *corev1.Pod {
 			Tolerations:       args.workloadNodePlacement.Tolerations,
 			Affinity:          args.workloadNodePlacement.Affinity,
 			PriorityClassName: args.priorityClassName,
+			ImagePullSecrets:  args.imagePullSecrets,
 		},
 	}
 
@@ -1002,7 +1009,7 @@ func makeNodeImporterPodSpec(args *importerPodArgs) *corev1.Pod {
 	args.podEnvVar.ep = "http://localhost:8100/disk.img"
 	args.podEnvVar.readyFile = "/shared/ready"
 	args.podEnvVar.doneFile = "/shared/done"
-	setImporterPodCommons(pod, args.podEnvVar, args.pvc, args.podResourceRequirements)
+	setImporterPodCommons(pod, args.podEnvVar, args.pvc, args.podResourceRequirements, args.imagePullSecrets)
 	pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
 		MountPath: "/shared",
 		Name:      "shared-volume",
@@ -1084,10 +1091,11 @@ func makeImporterPodSpec(args *importerPodArgs) *corev1.Pod {
 			Tolerations:       args.workloadNodePlacement.Tolerations,
 			Affinity:          args.workloadNodePlacement.Affinity,
 			PriorityClassName: args.priorityClassName,
+			ImagePullSecrets:  args.imagePullSecrets,
 		},
 	}
 
-	setImporterPodCommons(pod, args.podEnvVar, args.pvc, args.podResourceRequirements)
+	setImporterPodCommons(pod, args.podEnvVar, args.pvc, args.podResourceRequirements, args.imagePullSecrets)
 
 	if args.scratchPvcName != nil {
 		pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
@@ -1161,12 +1169,13 @@ func makeImporterPodSpec(args *importerPodArgs) *corev1.Pod {
 	return pod
 }
 
-func setImporterPodCommons(pod *corev1.Pod, podEnvVar *importPodEnvVar, pvc *corev1.PersistentVolumeClaim, podResourceRequirements *corev1.ResourceRequirements) {
+func setImporterPodCommons(pod *corev1.Pod, podEnvVar *importPodEnvVar, pvc *corev1.PersistentVolumeClaim, podResourceRequirements *corev1.ResourceRequirements, imagePullSecrets []corev1.LocalObjectReference) {
 	if podResourceRequirements != nil {
 		for i := range pod.Spec.Containers {
 			pod.Spec.Containers[i].Resources = *podResourceRequirements
 		}
 	}
+	pod.Spec.ImagePullSecrets = imagePullSecrets
 
 	ownerUID := pvc.UID
 	if len(pvc.OwnerReferences) == 1 {

diff --git a/pkg/controller/upload-controller.go b/pkg/controller/upload-controller.go
@@ -578,12 +578,17 @@ func (r *UploadReconciler) createUploadPod(args UploadPodArgs) (*v1.Pod, error)
 		return nil, err
 	}
 
+	imagePullSecrets, err := cc.GetImagePullSecrets(r.client)
+	if err != nil {
+		return nil, err
+	}
+
 	workloadNodePlacement, err := cc.GetWorkloadNodePlacement(r.client)
 	if err != nil {
 		return nil, err
 	}
 
-	pod := r.makeUploadPodSpec(args, podResourceRequirements, workloadNodePlacement)
+	pod := r.makeUploadPodSpec(args, podResourceRequirements, imagePullSecrets, workloadNodePlacement)
 	util.SetRecommendedLabels(pod, r.installerLabels, "cdi-controller")
 
 	if err := r.client.Get(context.TODO(), types.NamespacedName{Name: args.Name, Namespace: ns}, pod); err != nil {
@@ -729,7 +734,7 @@ func createUploadServiceNameFromPvcName(pvc string) string {
 	return naming.GetServiceNameFromResourceName(createUploadResourceName(pvc))
 }
 
-func (r *UploadReconciler) makeUploadPodSpec(args UploadPodArgs, resourceRequirements *v1.ResourceRequirements, workloadNodePlacement *sdkapi.NodePlacement) *v1.Pod {
+func (r *UploadReconciler) makeUploadPodSpec(args UploadPodArgs, resourceRequirements *v1.ResourceRequirements, imagePullSecrets []v1.LocalObjectReference, workloadNodePlacement *sdkapi.NodePlacement) *v1.Pod {
 	requestImageSize, _ := cc.GetRequestedImageSize(args.PVC)
 	serviceName := naming.GetServiceNameFromResourceName(args.Name)
 	pod := &v1.Pod{
@@ -839,6 +844,7 @@ func (r *UploadReconciler) makeUploadPodSpec(args UploadPodArgs, resourceRequire
 			Tolerations:       workloadNodePlacement.Tolerations,
 			Affinity:          workloadNodePlacement.Affinity,
 			PriorityClassName: cc.GetPriorityClass(args.PVC),
+			ImagePullSecrets:  imagePullSecrets,
 		},
 	}
 

diff --git a/pkg/operator/controller/controller_test.go b/pkg/operator/controller/controller_test.go
@@ -1244,7 +1244,9 @@ var _ = Describe("Controller", func() {
 
 			Entry("verify - unused deployment deleted",
 				func() (client.Object, error) {
-					deployment := utils.CreateDeployment("fake-cdi-deployment", "app", "containerized-data-importer", "fake-sa", int32(1), &sdkapi.NodePlacement{})
+					const imagePullSecretName = "fake-registry-key"
+					var imagePullSecrets = []corev1.LocalObjectReference{{Name: imagePullSecretName}}
+					deployment := utils.CreateDeployment("fake-cdi-deployment", "app", "containerized-data-importer", "fake-sa", imagePullSecrets, int32(1), &sdkapi.NodePlacement{})
 					return deployment, nil
 				}),
 			Entry("verify - unused service deleted",

diff --git a/pkg/operator/controller/cr-manager.go b/pkg/operator/controller/cr-manager.go
@@ -83,6 +83,9 @@ func (r *ReconcileCDI) getNamespacedArgs(cr *cdiv1.CDI) *cdinamespaced.FactoryAr
 		if cr.Spec.ImagePullPolicy != "" {
 			result.PullPolicy = string(cr.Spec.ImagePullPolicy)
 		}
+		if cr.Spec.Config != nil && len(cr.Spec.Config.ImagePullSecrets) > 0 {
+			result.ImagePullSecrets = cr.Spec.Config.ImagePullSecrets
+		}
 		if cr.Spec.PriorityClass != nil && string(*cr.Spec.PriorityClass) != "" {
 			result.PriorityClassName = string(*cr.Spec.PriorityClass)
 		} else {