From 1a0d30b26366169716a0c2ab3ede6100963fb45b Mon Sep 17 00:00:00 2001 From: Fred Rolland Date: Mon, 20 May 2024 12:16:12 +0300 Subject: [PATCH] feat: MOFED subscription mounts Add needed mounts for subscription in RHEL with non CRIO container runtime. Signed-off-by: Fred Rolland --- .../0050_ofed-driver-ds.yaml | 6 + pkg/nodeinfo/attributes.go | 7 + pkg/nodeinfo/node_info.go | 33 +++- pkg/state/state_ofed.go | 76 ++++++++++ pkg/state/state_ofed_test.go | 143 ++++++++++++++++++ 5 files changed, 259 insertions(+), 6 deletions(-) diff --git a/manifests/state-ofed-driver/0050_ofed-driver-ds.yaml b/manifests/state-ofed-driver/0050_ofed-driver-ds.yaml index 12898d18..84673bbb 100644 --- a/manifests/state-ofed-driver/0050_ofed-driver-ds.yaml +++ b/manifests/state-ofed-driver/0050_ofed-driver-ds.yaml @@ -237,12 +237,18 @@ spec: type: DirectoryOrCreate {{- range .AdditionalVolumeMounts.Volumes }} - name: {{ .Name }} + {{- if and .ConfigMap .ConfigMap.Items }} configMap: name: {{ .Name }} items: {{- range .ConfigMap.Items }} {{ . | yaml | nindentPrefix 14 "- " }} {{- end }} + {{- else if .HostPath }} + hostPath: + path: {{ .HostPath.Path }} + type: {{ .HostPath.Type }} + {{- end }} {{- end }} {{- if .RuntimeSpec.UseDtk }} - name: shared-doca-driver-toolkit diff --git a/pkg/nodeinfo/attributes.go b/pkg/nodeinfo/attributes.go index 2bbd8b7d..e19eaa8f 100644 --- a/pkg/nodeinfo/attributes.go +++ b/pkg/nodeinfo/attributes.go @@ -35,3 +35,10 @@ const ( NodeLabelCudaVersionMajor = "nvidia.com/cuda.driver.major" NodeLabelOSTreeVersion = "feature.node.kubernetes.io/system-os_release.OSTREE_VERSION" ) + +// Constants for Container Runtime +const ( + Docker string = "docker" + Containerd string = "containerd" + CRIO string = "cri-o" +) diff --git a/pkg/nodeinfo/node_info.go b/pkg/nodeinfo/node_info.go index 4b714c89..b37fd829 100644 --- a/pkg/nodeinfo/node_info.go +++ b/pkg/nodeinfo/node_info.go @@ -22,6 +22,7 @@ package nodeinfo import ( "fmt" + "strings" corev1 "k8s.io/api/core/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -51,12 +52,13 @@ type provider struct { // NodePool represent a set of Nodes grouped by common attributes type NodePool struct { - Name string - OsName string - OsVersion string - RhcosVersion string - Kernel string - Arch string + Name string + OsName string + OsVersion string + RhcosVersion string + Kernel string + Arch string + ContainerRuntime string } // GetNodePools partitions nodes into one or more node pools. The list of nodes to partition @@ -113,6 +115,8 @@ func (p *provider) GetNodePools(filters ...Filter) []NodePool { } nodePool.Kernel = kernel + nodePool.ContainerRuntime = getContainerRuntime(node) + nodePool.Name = fmt.Sprintf("%s%s-%s", nodePool.OsName, nodePool.OsVersion, nodePool.Kernel) if _, exists := nodePoolMap[nodePool.Name]; !exists { @@ -128,3 +132,20 @@ func (p *provider) GetNodePools(filters ...Filter) []NodePool { return nodePools } + +func getContainerRuntime(node *corev1.Node) string { + // runtimeVer string will look like :// + runtimeVer := node.Status.NodeInfo.ContainerRuntimeVersion + var runtime string + switch { + case strings.HasPrefix(runtimeVer, "docker"): + runtime = Docker + case strings.HasPrefix(runtimeVer, "containerd"): + runtime = Containerd + case strings.HasPrefix(runtimeVer, "cri-o"): + runtime = CRIO + default: + runtime = "" + } + return runtime +} diff --git a/pkg/state/state_ofed.go b/pkg/state/state_ofed.go index 2c592c63..b67ef4eb 100644 --- a/pkg/state/state_ofed.go +++ b/pkg/state/state_ofed.go @@ -120,6 +120,42 @@ var RepoConfigPathMap = map[string]string{ "rhel": "/etc/yum.repos.d", } +// MountPathToVolumeSource maps a container mount path to a VolumeSource +type MountPathToVolumeSource map[string]v1.VolumeSource + +// SubscriptionPathMap contains information on OS-specific paths +// that provide entitlements/subscription details on the host. +// These are used to enable Driver Container's access to packages controlled by +// the distro through their subscription and support program. +var SubscriptionPathMap = map[string]MountPathToVolumeSource{ + "rhel": { + "/run/secrets/etc-pki-entitlement": v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/pki/entitlement", + Type: newHostPathType(v1.HostPathDirectory), + }, + }, + "/run/secrets/redhat.repo": v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/yum.repos.d/redhat.repo", + Type: newHostPathType(v1.HostPathFile), + }, + }, + "/run/secrets/rhsm": v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/rhsm", + Type: newHostPathType(v1.HostPathDirectory), + }, + }, + }, +} + +func newHostPathType(pathType v1.HostPathType) *v1.HostPathType { + hostPathType := new(v1.HostPathType) + *hostPathType = pathType + return hostPathType +} + // ConfigMapKeysOverride contains static key override rules for ConfigMaps // now the only use-case is to override key name in the ConfigMap which automatically // populated by Openshift @@ -492,6 +528,12 @@ func renderObjects(ctx context.Context, nodePool *nodeinfo.NodePool, useDtk bool return nil, err } + // set subscription volumes if needed + err = s.handleSubscriptionVolumes(ctx, osname, nodePool.ContainerRuntime, &additionalVolMounts) + if err != nil { + return nil, err + } + renderData := &ofedManifestRenderData{ CrSpec: cr.Spec.OFEDDriver, RuntimeSpec: &ofedRuntimeSpec{ @@ -783,6 +825,40 @@ func (s *stateOFED) handleCertConfig( return nil } +// handleSubscriptionVolumes handles additional mounts required for subscriptions +func (s *stateOFED) handleSubscriptionVolumes( + ctx context.Context, osname string, runtime string, mounts *additionalVolumeMounts) error { + reqLogger := log.FromContext(ctx) + if osname == "rhel" && runtime != nodeinfo.CRIO { + reqLogger.V(consts.LogLevelDebug).Info("Setting subscription mounts for RHEL with non CRIO container runtime") + pathToVolumeSource, ok := SubscriptionPathMap[osname] + if !ok { + return fmt.Errorf("failed to find subscription volumes definition for os: %v", osname) + } + // sort host path volumes to ensure ordering is preserved when adding to pod spec + mountPaths := make([]string, 0, len(pathToVolumeSource)) + for k := range pathToVolumeSource { + mountPaths = append(mountPaths, k) + } + sort.Strings(mountPaths) + + for num, mountPath := range mountPaths { + volMountSubscriptionName := fmt.Sprintf("subscription-config-%d", num) + + volMountSubscription := v1.VolumeMount{ + Name: volMountSubscriptionName, + MountPath: mountPath, + ReadOnly: true, + } + mounts.VolumeMounts = append(mounts.VolumeMounts, volMountSubscription) + + subscriptionVol := v1.Volume{Name: volMountSubscriptionName, VolumeSource: pathToVolumeSource[mountPath]} + mounts.Volumes = append(mounts.Volumes, subscriptionVol) + } + } + return nil +} + // handleRepoConfig handles additional mounts required for custom repo if specified func (s *stateOFED) handleRepoConfig( ctx context.Context, cr *mellanoxv1alpha1.NicClusterPolicy, osname string, mounts *additionalVolumeMounts) error { diff --git a/pkg/state/state_ofed_test.go b/pkg/state/state_ofed_test.go index a3a06ec4..ffad9b85 100644 --- a/pkg/state/state_ofed_test.go +++ b/pkg/state/state_ofed_test.go @@ -326,6 +326,59 @@ var _ = Describe("MOFED state test", func() { verifyPodAntiInfinity(ds.Spec.Template.Spec.Affinity) } }) + It("Should Render subscription mounts for RHEL + containerd", func() { + client := mocks.ControllerRuntimeClient{} + manifestBaseDir := "../../manifests/state-ofed-driver" + + files, err := utils.GetFilesWithSuffix(manifestBaseDir, render.ManifestFileSuffix...) + Expect(err).NotTo(HaveOccurred()) + renderer := render.NewRenderer(files) + + ofedState := stateOFED{ + stateSkel: stateSkel{ + name: stateOFEDName, + description: stateOFEDDescription, + client: &client, + renderer: renderer, + }, + } + cr := &v1alpha1.NicClusterPolicy{} + cr.Name = "nic-cluster-policy" + cr.Spec.OFEDDriver = &v1alpha1.OFEDDriverSpec{ + ImageSpec: v1alpha1.ImageSpec{ + Image: "mofed", + Repository: "nvcr.io/mellanox", + Version: "23.10-0.5.5.0", + }, + } + + By("Creating NodeProvider with 1 Nodes, RHEL with containerd") + node := getNode("node1", kernelFull1) + setContainerRuntime(node, "containerd://1.27.1") + node.Labels[nodeinfo.NodeLabelOSName] = "rhel" + infoProvider := nodeinfo.NewProvider([]*v1.Node{ + node, + }) + catalog := NewInfoCatalog() + catalog.Add(InfoTypeClusterType, &dummyProvider{}) + catalog.Add(InfoTypeNodeInfo, infoProvider) + catalog.Add(InfoTypeDocaDriverImage, &dummyOfedImageProvider{tagExists: false}) + objs, err := ofedState.GetManifestObjects(ctx, cr, catalog, testLogger) + Expect(err).NotTo(HaveOccurred()) + // Expect 5 objects: 1 DS per pool, Service Account, Role, RoleBinding + Expect(len(objs)).To(Equal(4)) + By("Verify Subscription mounts") + for _, obj := range objs { + if obj.GetKind() != "DaemonSet" { + continue + } + ds := appsv1.DaemonSet{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj.Object, &ds) + Expect(err).NotTo(HaveOccurred()) + verifySubscriptionMounts(ds.Spec.Template.Spec.Containers[0].VolumeMounts) + verifySubscriptionVolumes(ds.Spec.Template.Spec.Volumes) + } + }) }) Context("Render Manifests DTK", func() { It("Should Render DaemonSet with DTK and additional mounts", func() { @@ -584,6 +637,88 @@ func verifyPodAntiInfinity(affinity *v1.Affinity) { Expect(*affinity).To(BeEquivalentTo(expected)) } +func verifySubscriptionMounts(mounts []v1.VolumeMount) { + By("Verify Subscription Mounts") + sub0 := v1.VolumeMount{ + Name: "subscription-config-0", + ReadOnly: true, + MountPath: "/run/secrets/etc-pki-entitlement", + SubPath: "", + MountPropagation: nil, + SubPathExpr: "", + } + Expect(slices.Contains(mounts, sub0)).To(BeTrue()) + sub1 := v1.VolumeMount{ + Name: "subscription-config-1", + ReadOnly: true, + MountPath: "/run/secrets/redhat.repo", + SubPath: "", + MountPropagation: nil, + SubPathExpr: "", + } + Expect(slices.Contains(mounts, sub1)).To(BeTrue()) + sub2 := v1.VolumeMount{ + Name: "subscription-config-2", + ReadOnly: true, + MountPath: "/run/secrets/rhsm", + SubPath: "", + MountPropagation: nil, + SubPathExpr: "", + } + Expect(slices.Contains(mounts, sub2)).To(BeTrue()) +} + +func verifySubscriptionVolumes(volumes []v1.Volume) { + By("Verify Subscription Volumes") + sub0 := v1.Volume{ + Name: "subscription-config-0", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/pki/entitlement", + Type: newHostPathType(v1.HostPathDirectory), + }, + }, + } + sub1 := v1.Volume{ + Name: "subscription-config-1", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/yum.repos.d/redhat.repo", + Type: newHostPathType(v1.HostPathFile), + }, + }, + } + sub2 := v1.Volume{ + Name: "subscription-config-2", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/etc/rhsm", + Type: newHostPathType(v1.HostPathDirectory), + }, + }, + } + foundSub0 := false + foundSub1 := false + foundSub2 := false + for i := range volumes { + if volumes[i].Name == "subscription-config-0" { + Expect(volumes[i]).To(BeEquivalentTo(sub0)) + foundSub0 = true + } + if volumes[i].Name == "subscription-config-1" { + Expect(volumes[i]).To(BeEquivalentTo(sub1)) + foundSub1 = true + } + if volumes[i].Name == "subscription-config-2" { + Expect(volumes[i]).To(BeEquivalentTo(sub2)) + foundSub2 = true + } + } + Expect(foundSub0).To(BeTrue()) + Expect(foundSub1).To(BeTrue()) + Expect(foundSub2).To(BeTrue()) +} + func verifyAdditionalMounts(mounts []v1.VolumeMount) { By("Verify Additional Mounts") repo := v1.VolumeMount{ @@ -686,3 +821,11 @@ func getNode(name, kernelFull string) *v1.Node { }, } } + +func setContainerRuntime(node *v1.Node, containerRuntime string) { + node.Status = v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + ContainerRuntimeVersion: containerRuntime, + }, + } +}