Skip to content

Commit

Permalink
fix: DTK additional mounts (Mellanox#842)
Browse files Browse the repository at this point in the history 
- Add additional mounts to DTK container
- Fix additional mounts handling issue introduced in Mellanox#707
  • Loading branch information
@e0ne
e0ne committed Mar 6, 2024
2 parents ee75a42 + 63502df commit e6cfb4d
Show file tree
Hide file tree
Showing 3 changed files with 113 additions and 8 deletions.
8 changes: 8 additions & 0 deletions manifests/state-ofed-driver/0050_ofed-driver-ds.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,14 @@ spec:
{{- end }}
{{- end }}
volumeMounts:
{{- if.AdditionalVolumeMounts.VolumeMounts }}
{{- range .AdditionalVolumeMounts.VolumeMounts }}
- name: {{ .Name }}
mountPath: {{ .MountPath }}
subPath: {{ .SubPath }}
readOnly: {{ .ReadOnly }}
{{- end }}
{{- end }}
- name: shared-doca-driver-toolkit
mountPath: /mnt/shared-doca-driver-toolkit
{{- with index .RuntimeSpec.ContainerResources "openshift-driver-toolkit-ctr" }}
Expand Down
12 changes: 6 additions & 6 deletions pkg/state/state_ofed.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -437,13 +437,13 @@ func (s *stateOFED) GetManifestObjects(
additionalVolMounts := additionalVolumeMounts{}
osname := nodeAttr[nodeinfo.AttrTypeOSName]
// set any custom ssl key/certificate configuration provided
err := s.handleCertConfig(ctx, cr, osname, additionalVolMounts)
err := s.handleCertConfig(ctx, cr, osname, &additionalVolMounts)
if err != nil {
return nil, err
}

// set any custom repo configuration provided
err = s.handleRepoConfig(ctx, cr, osname, additionalVolMounts)
err = s.handleRepoConfig(ctx, cr, osname, &additionalVolMounts)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -705,14 +705,14 @@ func setProbesDefaults(cr *mellanoxv1alpha1.NicClusterPolicy) {

// handleCertConfig handles additional mounts required for Certificates if specified
func (s *stateOFED) handleCertConfig(
ctx context.Context, cr *mellanoxv1alpha1.NicClusterPolicy, osname string, mounts additionalVolumeMounts) error {
ctx context.Context, cr *mellanoxv1alpha1.NicClusterPolicy, osname string, mounts *additionalVolumeMounts) error {
if cr.Spec.OFEDDriver.CertConfig != nil && cr.Spec.OFEDDriver.CertConfig.Name != "" {
destinationDir, err := getCertConfigPath(osname)
if err != nil {
return fmt.Errorf("failed to get destination directory for custom TLS certificates config: %v", err)
}

err = s.handleAdditionalMounts(ctx, &mounts, cr.Spec.OFEDDriver.CertConfig.Name, destinationDir)
err = s.handleAdditionalMounts(ctx, mounts, cr.Spec.OFEDDriver.CertConfig.Name, destinationDir)
if err != nil {
return fmt.Errorf("failed to mount volumes for custom TLS certificates: %v", err)
}
Expand All @@ -722,14 +722,14 @@ func (s *stateOFED) handleCertConfig(

// handleRepoConfig handles additional mounts required for custom repo if specified
func (s *stateOFED) handleRepoConfig(
ctx context.Context, cr *mellanoxv1alpha1.NicClusterPolicy, osname string, mounts additionalVolumeMounts) error {
ctx context.Context, cr *mellanoxv1alpha1.NicClusterPolicy, osname string, mounts *additionalVolumeMounts) error {
if cr.Spec.OFEDDriver.RepoConfig != nil && cr.Spec.OFEDDriver.RepoConfig.Name != "" {
destinationDir, err := getRepoConfigPath(osname)
if err != nil {
return fmt.Errorf("failed to get destination directory for custom repo config: %v", err)
}

err = s.handleAdditionalMounts(ctx, &mounts, cr.Spec.OFEDDriver.RepoConfig.Name, destinationDir)
err = s.handleAdditionalMounts(ctx, mounts, cr.Spec.OFEDDriver.RepoConfig.Name, destinationDir)
if err != nil {
return fmt.Errorf("failed to mount volumes for custom repositories configuration: %v", err)
}
Expand Down
101 changes: 99 additions & 2 deletions pkg/state/state_ofed_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package state

import (
"context"
"slices"
"strings"

"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -292,7 +293,7 @@ var _ = Describe("MOFED state test", func() {
})
})
Context("Render Manifests DTK", func() {
It("Should Render DaemonSet with DTK", func() {
It("Should Render DaemonSet with DTK and additional mounts", func() {
dtkImageName := "quay.io/openshift-release-dev/ocp-v4.0-art-dev:414"
dtkImageStream := &apiimagev1.ImageStream{
TypeMeta: metav1.TypeMeta{
Expand All @@ -315,9 +316,24 @@ var _ = Describe("MOFED state test", func() {
},
},
}
cmRepo := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "repo-cm",
Namespace: "nvidia-network-operator",
},
Data: map[string]string{"ubi.repo": "somerepocontents"},
}
cmCert := &v1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: "cert-cm",
Namespace: "nvidia-network-operator",
},
Data: map[string]string{"my-cert": "somecertificate"},
}
scheme := runtime.NewScheme()
Expect(v1.AddToScheme(scheme)).NotTo(HaveOccurred())
Expect(apiimagev1.AddToScheme(scheme)).NotTo(HaveOccurred())
client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(dtkImageStream).Build()
client := fake.NewClientBuilder().WithScheme(scheme).WithObjects(dtkImageStream, cmRepo, cmCert).Build()
manifestBaseDir := "../../manifests/state-ofed-driver"

files, err := utils.GetFilesWithSuffix(manifestBaseDir, render.ManifestFileSuffix...)
Expand All @@ -340,6 +356,12 @@ var _ = Describe("MOFED state test", func() {
Repository: "nvcr.io/mellanox",
Version: "23.10-0.5.5.0",
},
RepoConfig: &v1alpha1.ConfigMapNameReference{
Name: "repo-cm",
},
CertConfig: &v1alpha1.ConfigMapNameReference{
Name: "cert-cm",
},
Env: []v1.EnvVar{
{
Name: "ENTRYPOINT_DEBUG",
Expand Down Expand Up @@ -375,6 +397,9 @@ var _ = Describe("MOFED state test", func() {
Expect(len(ds.Spec.Template.Spec.Containers)).To(Equal(2))
dtkContainer := ds.Spec.Template.Spec.Containers[1]
Expect(dtkContainer.Image).To(Equal(dtkImageName))
verifyAdditionalMounts(ds.Spec.Template.Spec.Containers[0].VolumeMounts)
verifyAdditionalMounts(ds.Spec.Template.Spec.Containers[1].VolumeMounts)
verifyAdditionalVolumes(ds.Spec.Template.Spec.Volumes)
}
})
})
Expand Down Expand Up @@ -403,6 +428,78 @@ func verifyPodAntiInfinity(affinity *v1.Affinity) {
Expect(*affinity).To(BeEquivalentTo(expected))
}

func verifyAdditionalMounts(mounts []v1.VolumeMount) {
By("Verify Additional Mounts")
repo := v1.VolumeMount{
Name: "repo-cm",
ReadOnly: true,
MountPath: "/etc/apt/sources.list.d/ubi.repo",
SubPath: "ubi.repo",
MountPropagation: nil,
SubPathExpr: "",
}
Expect(slices.Contains(mounts, repo)).To(BeTrue())
cert := v1.VolumeMount{
Name: "cert-cm",
ReadOnly: true,
MountPath: "/etc/ssl/certs/my-cert",
SubPath: "my-cert",
MountPropagation: nil,
SubPathExpr: "",
}
Expect(slices.Contains(mounts, cert)).To(BeTrue())
}

func verifyAdditionalVolumes(volumes []v1.Volume) {
By("Verify Additional Volumes")
certVol := v1.Volume{
Name: "cert-cm",
VolumeSource: v1.VolumeSource{
ConfigMap: &v1.ConfigMapVolumeSource{
LocalObjectReference: v1.LocalObjectReference{
Name: "cert-cm",
},
Items: []v1.KeyToPath{
{
Key: "my-cert",
Path: "my-cert",
},
},
},
},
}
repoVol := v1.Volume{
Name: "repo-cm",
VolumeSource: v1.VolumeSource{
ConfigMap: &v1.ConfigMapVolumeSource{
LocalObjectReference: v1.LocalObjectReference{
Name: "repo-cm",
},
Items: []v1.KeyToPath{
{
Key: "ubi.repo",
Path: "ubi.repo",
},
},
},
},
}
foundCert := false
foundRepo := false
for i := range volumes {
if volumes[i].Name == "cert-cm" {
Expect(volumes[i]).To(BeEquivalentTo(certVol))
foundCert = true
}
if volumes[i].Name == "repo-cm" {
Expect(volumes[i]).To(BeEquivalentTo(repoVol))
foundRepo = true
}
}
Expect(foundCert).To(BeTrue())
Expect(foundRepo).To(BeTrue())
}

func verifyDSNodeSelector(selector map[string]string) {
By("Verify NodeSelector")
nsMellanox, ok := selector["feature.node.kubernetes.io/pci-15b3.present"]
Expand Down

0 comments on commit e6cfb4d

Please sign in to comment.