From a1d6c12674e2a6d3ed21bafc45aec2185ae8a1c0 Mon Sep 17 00:00:00 2001 From: Jason Davis Date: Wed, 18 Sep 2024 11:58:48 -0600 Subject: [PATCH 1/3] MTL-1832 --- pkg/cli/config/initialize/basecamp.go | 90 ++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 7 deletions(-) diff --git a/pkg/cli/config/initialize/basecamp.go b/pkg/cli/config/initialize/basecamp.go index 3fa0ac0da..fa09b2e4b 100644 --- a/pkg/cli/config/initialize/basecamp.go +++ b/pkg/cli/config/initialize/basecamp.go @@ -119,6 +119,73 @@ type BaseCampGlobals struct { } // Basecamp Defaults +// master bootcmd (cloud-init user-data) +var masterBootCMD = [][]string{ + {"cloud-init-per", "once", "create_PART_LVM", "parted", "--wipesignatures", "-m", "--align=opt", "--ignore-busy", "/dev/md/AUX", "mkpart", "primary", "1MiB", "100%"}, + {"cloud-init-per", "once", "create_PV", "pvcreate", "-ff", "-y", "-M", "lvm2", "/dev/md/AUX1"}, + {"cloud-init-per", "once", "create_VG", "vgcreate", "metalvg0", "/dev/md/AUX1"}, + {"cloud-init-per", "once", "create_LV_CRAYS3CACHE", "lvcreate", "-l", "25%PVS", "-n", "CRAYS3CACHE", "-y", "metalvg0"}, + {"cloud-init-per", "once", "create_LV_CONRUN", "lvcreate", "-l", "4%PVS", "-n", "CONRUN", "-y", "metalvg0"}, + {"cloud-init-per", "once", "create_LV_CONLIB", "lvcreate", "-l", "36%PVS", "-n", "CONLIB", "-y", "metalvg0"}, + {"cloud-init-per", "once", "create_LV_K8SLET", "lvcreate", "-l", "10%PVS", "-n", "K8SLET", "-y", "metalvg0"}, +} + +// master fs_setup (cloud-init user-data) +var masterFileSystems = []map[string]interface{}{ + {"label": "CRAYS3CACHE", "filesystem": "ext4", "device": "/dev/disk/by-id/dm-name-metalvg0-CRAYS3CACHE", "partition": "auto", "overwrite": true}, + {"label": "CONRUN", "filesystem": "xfs", "device": "/dev/disk/by-id/dm-name-metalvg0-CONRUN", "partition": "auto", "overwrite": true}, + {"label": "CONLIB", "filesystem": "xfs", "device": "/dev/disk/by-id/dm-name-metalvg0-CONLIB", "partition": "auto", "overwrite": true}, + {"label": "K8SLET", "filesystem": "xfs", "device": "/dev/disk/by-id/dm-name-metalvg0-K8SLET", "partition": "auto", "overwrite": true}, +} + +// master mounts (cloud-init user-data) +var masterMounts = [][]string{ + {"LABEL=CRAYS3CACHE", "/var/lib/s3fs_cache", "ext4", "defaults,nofail"}, + {"LABEL=CONRUN", "/run/containerd", "xfs", "defaults,nofail"}, + {"LABEL=CONLIB", "/var/lib/containerd", "xfs", "defaults,nofail"}, + {"LABEL=K8SLET", "/var/lib/kubelet", "xfs", "defaults,nofail"}, +} + +// worker bootcmd (cloud-init user-data) +var workerBootCMD = [][]string{ + {"cloud-init-per", "once", "create_PV", "pvcreate", "-ff", "-y", "-M", "lvm2", "/dev/md/AUX"}, + {"cloud-init-per", "once", "create_VG", "vgcreate", "metalvg0", "/dev/md/AUX"}, + {"cloud-init-per", "once", "create_LV_CRAYS3CACHE", "lvcreate", "-L", "200GB", "-n", "CRAYS3CACHE", "-y", "metalvg0"}, +} + +// worker fs_setup (cloud-init user-data) +var workerFileSystems = []map[string]interface{}{ + {"label": "CRAYS3CACHE", "filesystem": "ext4", "device": "/dev/disk/by-id/dm-name-metalvg0-CRAYS3CACHE", "partition": "auto", "overwrite": true}, +} + +// worker mounts (cloud-init user-data) +var workerMounts = [][]string{ + {"LABEL=CRAYS3CACHE", "/var/lib/s3fs_cache", "ext4", "defaults,nofail"}, +} + +// storage bootcmd (cloud-init user-data) +var cephBootCMD = [][]string{ + {"cloud-init-per", "once", "create_PV", "pvcreate", "-ff", "-y", "-M", "lvm2", "/dev/md/AUX"}, + {"cloud-init-per", "once", "create_VG", "vgcreate", "metalvg0", "/dev/md/AUX"}, + {"cloud-init-per", "once", "create_LV_CEPHETC", "lvcreate", "-L", "10GB", "-n", "CEPHETC", "-y", "metalvg0"}, + {"cloud-init-per", "once", "create_LV_CEPHVAR", "lvcreate", "-L", "60GB", "-n", "CEPHVAR", "-y", "metalvg0"}, + {"cloud-init-per", "once", "create_LV_CONTAIN", "lvcreate", "-L", "60GB", "-n", "CONTAIN", "-y", "metalvg0"}, +} + +// storage fs_setup (cloud-init user-data) +var cephFileSystems = []map[string]interface{}{ + {"label": "CEPHETC", "filesystem": "ext4", "device": "/dev/disk/by-id/dm-name-metalvg0-CEPHETC", "partition": "auto", "overwrite": true}, + {"label": "CEPHVAR", "filesystem": "ext4", "device": "/dev/disk/by-id/dm-name-metalvg0-CEPHVAR", "partition": "auto", "overwrite": true}, + {"label": "CONTAIN", "filesystem": "xfs", "device": "/dev/disk/by-id/dm-name-metalvg0-CONTAIN", "partition": "auto", "overwrite": true}, +} + +// storage mounts (cloud-init user-data) +var cephMounts = [][]string{ + {"LABEL=CEPHETC", "/etc/ceph", "auto", "defaults"}, + {"LABEL=CEPHVAR", "/var/lib/ceph", "auto", "defaults"}, + {"LABEL=CONTAIN", "/var/lib/containers", "auto", "defaults"}, +} + // We should try to make these customizable by the user at some point // k8sRunCMD has the list of scripts to run on NCN boot for // all members of the kubernetes cluster @@ -603,19 +670,28 @@ func MakeBaseCampfromNCNs( ShastaRole: "ncn-" + strings.ToLower(ncn.Subrole), IPAM: ncnIPAM, } + userDataMap := make(map[string]interface{}) - if ncn.Subrole == "Storage" { - if strings.HasSuffix( - ncn.Hostname, - "001", - ) { + switch ncn.Subrole { + case "Storage": + userDataMap["bootcmd"] = cephBootCMD + userDataMap["fs_setup"] = cephFileSystems + userDataMap["mounts"] = cephMounts + if strings.HasSuffix(ncn.Hostname, "001") { userDataMap["runcmd"] = cephRunCMD } else { userDataMap["runcmd"] = cephWorkerRunCMD } - } else { - userDataMap["runcmd"] = k8sRunCMD + case "Master": + userDataMap["bootcmd"] = masterBootCMD + userDataMap["fs_setup"] = masterFileSystems + userDataMap["mounts"] = masterMounts + case "Worker": + userDataMap["bootcmd"] = workerBootCMD + userDataMap["fs_setup"] = workerFileSystems + userDataMap["mounts"] = workerMounts } + userDataMap["hostname"] = ncn.Hostname userDataMap["local_hostname"] = ncn.Hostname userDataMap["mac0"] = mac0Interface From 0ab900728d5c7661323d8dd8644cfaa6afe9e742 Mon Sep 17 00:00:00 2001 From: Jason Davis Date: Fri, 20 Sep 2024 08:41:02 -0600 Subject: [PATCH 2/3] move disks into separate file --- pkg/cli/config/initialize/basecamp.go | 67 +----- pkg/cli/config/initialize/disks.go | 308 ++++++++++++++++++++++++++ 2 files changed, 309 insertions(+), 66 deletions(-) create mode 100644 pkg/cli/config/initialize/disks.go diff --git a/pkg/cli/config/initialize/basecamp.go b/pkg/cli/config/initialize/basecamp.go index fa09b2e4b..79086eb28 100644 --- a/pkg/cli/config/initialize/basecamp.go +++ b/pkg/cli/config/initialize/basecamp.go @@ -119,72 +119,7 @@ type BaseCampGlobals struct { } // Basecamp Defaults -// master bootcmd (cloud-init user-data) -var masterBootCMD = [][]string{ - {"cloud-init-per", "once", "create_PART_LVM", "parted", "--wipesignatures", "-m", "--align=opt", "--ignore-busy", "/dev/md/AUX", "mkpart", "primary", "1MiB", "100%"}, - {"cloud-init-per", "once", "create_PV", "pvcreate", "-ff", "-y", "-M", "lvm2", "/dev/md/AUX1"}, - {"cloud-init-per", "once", "create_VG", "vgcreate", "metalvg0", "/dev/md/AUX1"}, - {"cloud-init-per", "once", "create_LV_CRAYS3CACHE", "lvcreate", "-l", "25%PVS", "-n", "CRAYS3CACHE", "-y", "metalvg0"}, - {"cloud-init-per", "once", "create_LV_CONRUN", "lvcreate", "-l", "4%PVS", "-n", "CONRUN", "-y", "metalvg0"}, - {"cloud-init-per", "once", "create_LV_CONLIB", "lvcreate", "-l", "36%PVS", "-n", "CONLIB", "-y", "metalvg0"}, - {"cloud-init-per", "once", "create_LV_K8SLET", "lvcreate", "-l", "10%PVS", "-n", "K8SLET", "-y", "metalvg0"}, -} - -// master fs_setup (cloud-init user-data) -var masterFileSystems = []map[string]interface{}{ - {"label": "CRAYS3CACHE", "filesystem": "ext4", "device": "/dev/disk/by-id/dm-name-metalvg0-CRAYS3CACHE", "partition": "auto", "overwrite": true}, - {"label": "CONRUN", "filesystem": "xfs", "device": "/dev/disk/by-id/dm-name-metalvg0-CONRUN", "partition": "auto", "overwrite": true}, - {"label": "CONLIB", "filesystem": "xfs", "device": "/dev/disk/by-id/dm-name-metalvg0-CONLIB", "partition": "auto", "overwrite": true}, - {"label": "K8SLET", "filesystem": "xfs", "device": "/dev/disk/by-id/dm-name-metalvg0-K8SLET", "partition": "auto", "overwrite": true}, -} - -// master mounts (cloud-init user-data) -var masterMounts = [][]string{ - {"LABEL=CRAYS3CACHE", "/var/lib/s3fs_cache", "ext4", "defaults,nofail"}, - {"LABEL=CONRUN", "/run/containerd", "xfs", "defaults,nofail"}, - {"LABEL=CONLIB", "/var/lib/containerd", "xfs", "defaults,nofail"}, - {"LABEL=K8SLET", "/var/lib/kubelet", "xfs", "defaults,nofail"}, -} - -// worker bootcmd (cloud-init user-data) -var workerBootCMD = [][]string{ - {"cloud-init-per", "once", "create_PV", "pvcreate", "-ff", "-y", "-M", "lvm2", "/dev/md/AUX"}, - {"cloud-init-per", "once", "create_VG", "vgcreate", "metalvg0", "/dev/md/AUX"}, - {"cloud-init-per", "once", "create_LV_CRAYS3CACHE", "lvcreate", "-L", "200GB", "-n", "CRAYS3CACHE", "-y", "metalvg0"}, -} - -// worker fs_setup (cloud-init user-data) -var workerFileSystems = []map[string]interface{}{ - {"label": "CRAYS3CACHE", "filesystem": "ext4", "device": "/dev/disk/by-id/dm-name-metalvg0-CRAYS3CACHE", "partition": "auto", "overwrite": true}, -} - -// worker mounts (cloud-init user-data) -var workerMounts = [][]string{ - {"LABEL=CRAYS3CACHE", "/var/lib/s3fs_cache", "ext4", "defaults,nofail"}, -} - -// storage bootcmd (cloud-init user-data) -var cephBootCMD = [][]string{ - {"cloud-init-per", "once", "create_PV", "pvcreate", "-ff", "-y", "-M", "lvm2", "/dev/md/AUX"}, - {"cloud-init-per", "once", "create_VG", "vgcreate", "metalvg0", "/dev/md/AUX"}, - {"cloud-init-per", "once", "create_LV_CEPHETC", "lvcreate", "-L", "10GB", "-n", "CEPHETC", "-y", "metalvg0"}, - {"cloud-init-per", "once", "create_LV_CEPHVAR", "lvcreate", "-L", "60GB", "-n", "CEPHVAR", "-y", "metalvg0"}, - {"cloud-init-per", "once", "create_LV_CONTAIN", "lvcreate", "-L", "60GB", "-n", "CONTAIN", "-y", "metalvg0"}, -} - -// storage fs_setup (cloud-init user-data) -var cephFileSystems = []map[string]interface{}{ - {"label": "CEPHETC", "filesystem": "ext4", "device": "/dev/disk/by-id/dm-name-metalvg0-CEPHETC", "partition": "auto", "overwrite": true}, - {"label": "CEPHVAR", "filesystem": "ext4", "device": "/dev/disk/by-id/dm-name-metalvg0-CEPHVAR", "partition": "auto", "overwrite": true}, - {"label": "CONTAIN", "filesystem": "xfs", "device": "/dev/disk/by-id/dm-name-metalvg0-CONTAIN", "partition": "auto", "overwrite": true}, -} - -// storage mounts (cloud-init user-data) -var cephMounts = [][]string{ - {"LABEL=CEPHETC", "/etc/ceph", "auto", "defaults"}, - {"LABEL=CEPHVAR", "/var/lib/ceph", "auto", "defaults"}, - {"LABEL=CONTAIN", "/var/lib/containers", "auto", "defaults"}, -} +// See disks.go for disk layout, filesystems, and mounts // We should try to make these customizable by the user at some point // k8sRunCMD has the list of scripts to run on NCN boot for diff --git a/pkg/cli/config/initialize/disks.go b/pkg/cli/config/initialize/disks.go new file mode 100644 index 000000000..89f8e8236 --- /dev/null +++ b/pkg/cli/config/initialize/disks.go @@ -0,0 +1,308 @@ +package initialize + +import "fmt" + +const ( + crays3cache = "CRAYS3CACHE" + conrun = "CONRUN" + conlib = "CONLIB" + k8slet = "K8SLET" + cephetc = "CEPHETC" + cephvar = "CEPHVAR" + contain = "CONTAIN" + volumeGroup = "metalvg0" + raidArray = "/dev/md/AUX" +) + +// master bootcmd (cloud-init user-data) +var masterBootCMD = [][]string{ + { + "cloud-init-per", + "once", + "create_PV", + "pvcreate", + "-ff", + "-y", + "-M", + "lvm2", + raidArray, + }, + { + "cloud-init-per", + "once", + "create_VG", + "vgcreate", + volumeGroup, + raidArray, + }, + { + "cloud-init-per", + "once", + fmt.Sprintf("create_LV_%s", crays3cache), + "lvcreate", + "-l", + "25%PVS", + "-n", + crays3cache, + "-y", + volumeGroup, + }, + { + "cloud-init-per", + "once", + fmt.Sprintf("create_LV_%s", conrun), + "lvcreate", + "-l", + "4%PVS", + "-n", + conrun, + "-y", + volumeGroup, + }, + { + "cloud-init-per", + "once", + fmt.Sprintf("create_LV_%s", conlib), + "lvcreate", + "-l", + "36%PVS", + "-n", + conlib, + "-y", + volumeGroup, + }, + { + "cloud-init-per", + "once", + fmt.Sprintf("create_LV_%s", k8slet), + "lvcreate", + "-l", + "10%PVS", + "-n", + k8slet, + "-y", + volumeGroup, + }, +} + +// master fs_setup (cloud-init user-data) +var masterFileSystems = []map[string]interface{}{ + { + "label": crays3cache, + "filesystem": "ext4", + "device": fmt.Sprintf("/dev/disk/by-id/dm-name-%s-%s", volumeGroup, crays3cache), + "partition": "auto", + "overwrite": true, + }, + { + "label": conrun, + "filesystem": "xfs", + "device": fmt.Sprintf("/dev/disk/by-id/dm-name-%s-%s", volumeGroup, conrun), + "partition": "auto", + "overwrite": true, + }, + { + "label": conlib, + "filesystem": "xfs", + "device": fmt.Sprintf("/dev/disk/by-id/dm-name-%s-%s", volumeGroup, conlib), + "partition": "auto", + "overwrite": true, + }, + { + "label": k8slet, + "filesystem": "xfs", + "device": fmt.Sprintf("/dev/disk/by-id/dm-name-%s-%s", volumeGroup, k8slet), + "partition": "auto", + "overwrite": true, + }, +} + +// master mounts (cloud-init user-data) +var masterMounts = [][]string{ + { + fmt.Sprintf("LABEL=%s", crays3cache), + "/var/lib/s3fs_cache", + "ext4", + "defaults,nofail", + }, + { + fmt.Sprintf("LABEL=%s", conrun), + "/run/containerd", + "xfs", + "defaults,nofail", + }, + { + fmt.Sprintf("LABEL=%s", conlib), + "/var/lib/containerd", + "xfs", + "defaults,nofail", + }, + { + fmt.Sprintf("LABEL=%s", k8slet), + "/var/lib/kubelet", + "xfs", + "defaults,nofail", + }, +} + +// worker bootcmd (cloud-init user-data) +var workerBootCMD = [][]string{ + { + "cloud-init-per", + "once", + "create_PV", + "pvcreate", + "-ff", + "-y", + "-M", + "lvm2", + raidArray, + }, + { + "cloud-init-per", + "once", + "create_VG", + "vgcreate", + volumeGroup, + raidArray, + }, + { + "cloud-init-per", + "once", + fmt.Sprintf("create_LV_%s", crays3cache), + "lvcreate", + "-L", + "200GB", + "-n", + crays3cache, + "-y", + volumeGroup, + }, +} + +// worker fs_setup (cloud-init user-data) +var workerFileSystems = []map[string]interface{}{ + { + "label": crays3cache, + "filesystem": "ext4", + "device": fmt.Sprintf("/dev/disk/by-id/dm-name-%s-%s", volumeGroup, crays3cache), + "partition": "auto", + "overwrite": true, + }, +} + +// worker mounts (cloud-init user-data) +var workerMounts = [][]string{ + { + fmt.Sprintf("LABEL=%s", crays3cache), + "/var/lib/s3fs_cache", + "ext4", + "defaults,nofail", + }, +} + +// storage bootcmd (cloud-init user-data) +var cephBootCMD = [][]string{ + { + "cloud-init-per", + "once", + "create_PV", + "pvcreate", + "-ff", + "-y", + "-M", + "lvm2", + raidArray, + }, + { + "cloud-init-per", + "once", + "create_VG", + "vgcreate", + volumeGroup, + raidArray, + }, + { + "cloud-init-per", + "once", + fmt.Sprintf("create_LV_%s", cephetc), + "lvcreate", + "-L", + "10GB", + "-n", + cephetc, + "-y", + volumeGroup, + }, + { + "cloud-init-per", + "once", + fmt.Sprintf("create_LV_%s", cephvar), + "lvcreate", + "-L", + "60GB", + "-n", + cephvar, + "-y", + volumeGroup, + }, + { + "cloud-init-per", + "once", + fmt.Sprintf("create_LV_%s", contain), + "lvcreate", + "-L", + "60GB", + "-n", + contain, + "-y", + volumeGroup, + }, +} + +// storage fs_setup (cloud-init user-data) +var cephFileSystems = []map[string]interface{}{ + { + "label": cephetc, + "filesystem": "ext4", + "device": fmt.Sprintf("/dev/disk/by-id/dm-name-%s-%s", volumeGroup, cephetc), + "partition": "auto", + "overwrite": true, + }, + { + "label": cephvar, + "filesystem": "ext4", + "device": fmt.Sprintf("/dev/disk/by-id/dm-name-%s-%s", volumeGroup, cephvar), + "partition": "auto", + "overwrite": true, + }, + { + "label": contain, + "filesystem": "xfs", + "device": fmt.Sprintf("/dev/disk/by-id/dm-name-%s-%s", volumeGroup, contain), + "partition": "auto", + "overwrite": true, + }, +} + +// storage mounts (cloud-init user-data) +var cephMounts = [][]string{ + { + fmt.Sprintf("LABEL=%s", cephetc), + "/etc/ceph", + "auto", + "defaults", + }, + { + fmt.Sprintf("LABEL=%s", cephvar), + "/var/lib/ceph", + "auto", + "defaults", + }, + { + fmt.Sprintf("LABEL=%s", contain), + "/var/lib/containers", + "auto", + "defaults", + }, +} From e87c9de38991a0f59581497776be9bb6f6d37e3e Mon Sep 17 00:00:00 2001 From: Jason Davis Date: Fri, 20 Sep 2024 11:03:38 -0600 Subject: [PATCH 3/3] add copyright --- pkg/cli/config/initialize/disks.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pkg/cli/config/initialize/disks.go b/pkg/cli/config/initialize/disks.go index 89f8e8236..6019434de 100644 --- a/pkg/cli/config/initialize/disks.go +++ b/pkg/cli/config/initialize/disks.go @@ -1,7 +1,32 @@ +/* + MIT License + + (C) Copyright 2024 Hewlett Packard Enterprise Development LP + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR + OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. +*/ + package initialize import "fmt" +// Provides configuration for lvm, filesystems and mounts for NCNS. const ( crays3cache = "CRAYS3CACHE" conrun = "CONRUN"