MTL-1832

Cray-HPE · Sep 19, 2024 · 4e8ca19 · 4e8ca19
1 parent eb726df
commit 4e8ca19
Showing 1 changed file with 114 additions and 0 deletions.
diff --git a/upgrade/scripts/upgrade/prerequisites.sh b/upgrade/scripts/upgrade/prerequisites.sh
@@ -1222,6 +1222,120 @@ else
   echo "====> ${state_name} has been completed" | tee -a "${LOG_FILE}"
 fi
 
+state_name="UPDATE_BSS_DATA_MASTERS"
+state_recorded=$(is_state_recorded "${state_name}" "$(hostname)")
+if [[ ${state_recorded} == "0" && $(hostname) == "${PRIMARY_NODE}" ]]; then
+  echo "====> ${state_name} ..." | tee -a "${LOG_FILE}"
+  {
+    # cloud-init bootcmd for master nodes
+    MASTER_BOOT_CMD='[
+      ["cloud-init-per", "once", "create_PART_LVM", "parted", "--wipesignatures", "-m", "--align=opt", "--ignore-busy", "/dev/md/AUX", "mkpart", "primary", "1MiB", "100%"],
+      ["cloud-init-per", "once", "create_PV", "pvcreate", "-ff", "-y", "-M", "lvm2", "/dev/md/AUX1"],
+      ["cloud-init-per", "once", "create_VG", "vgcreate", "metalvg0", "/dev/md/AUX1"],
+      ["cloud-init-per", "once", "create_LV_CRAYS3CACHE", "lvcreate", "-l", "25%PVS", "-n", "CRAYS3CACHE", "-y", "metalvg0"],
+      ["cloud-init-per", "once", "create_LV_CONRUN", "lvcreate", "-l", "4%PVS", "-n", "CONRUN", "-y", "metalvg0"],
+      ["cloud-init-per", "once", "create_LV_CONLIB", "lvcreate", "-l", "36%PVS", "-n", "CONLIB", "-y", "metalvg0"],
+      ["cloud-init-per", "once", "create_LV_K8SLET", "lvcreate", "-l", "10%PVS", "-n", "K8SLET", "-y", "metalvg0"]
+    ]'
+
+    # cloud-init fs_setup
+    MASTER_FILE_SYSTEMS='[
+      {"label": "CRAYS3CACHE", "filesystem": "ext4", "device": "/dev/disk/by-id/dm-name-metalvg0-CRAYS3CACHE", "partition": "auto", "overwrite": true},
+      {"label": "CONRUN", "filesystem": "xfs", "device": "/dev/disk/by-id/dm-name-metalvg0-CONRUN", "partition": "auto", "overwrite": true},
+      {"label": "CONLIB", "filesystem": "xfs", "device": "/dev/disk/by-id/dm-name-metalvg0-CONLIB", "partition": "auto", "overwrite": true},
+      {"label": "K8SLET", "filesystem": "xfs", "device": "/dev/disk/by-id/dm-name-metalvg0-K8SLET", "partition": "auto", "overwrite": true}
+    ]'
+
+    # cloud-init mounts
+    MASTER_MOUNTS='[
+      ["LABEL=CRAYS3CACHE", "/var/lib/s3fs_cache", "ext4", "defaults,nofail"],
+      ["LABEL=CONRUN", "/run/containerd", "xfs", "defaults,nofail"],
+      ["LABEL=CONLIB", "/var/lib/containerd", "xfs", "defaults,nofail"],
+      ["LABEL=K8SLET", "/var/lib/kubelet", "xfs", "defaults,nofail"]
+    ]'
+
+    # cloud-init bootcmd for worker nodes
+    WORKER_BOOT_CMD='[
+      ["cloud-init-per", "once", "create_PV", "pvcreate", "-ff", "-y", "-M", "lvm2", "/dev/md/AUX"], 
+      ["cloud-init-per", "once", "create_VG", "vgcreate", "metalvg0", "/dev/md/AUX"],
+      ["cloud-init-per", "once", "create_LV_CRAYS3CACHE", "lvcreate", "-L", "200GB", "-n", "CRAYS3CACHE", "-y", "metalvg0"]
+    ]'
+
+    # cloud-init fs_setup for worker nodes
+    WORKER_FILE_SYSTEMS='[
+      {"label": "CRAYS3CACHE", "filesystem": "ext4", "device": "/dev/disk/by-id/dm-name-metalvg0-CRAYS3CACHE", "partition": "auto", "overwrite": true}
+    ]'
+
+    # cloud-init mounts for worker nodes
+    WORKER_MOUNTS='[
+      ["LABEL=CRAYS3CACHE", "/var/lib/s3fs_cache", "ext4", "defaults,nofail"],
+    ]'
+
+    # cloud-init bootcmd for storage nodes
+    STORAGE_BOOT_CMD='[
+      ["cloud-init-per", "once", "create_PV", "pvcreate", "-ff", "-y", "-M", "lvm2", "/dev/md/AUX"],
+      ["cloud-init-per", "once", "create_VG", "vgcreate", "metalvg0", "/dev/md/AUX"],
+      ["cloud-init-per", "once", "create_LV_CEPHETC", "lvcreate", "-L", "10GB", "-n", "CEPHETC", "-y", "metalvg0"],
+      ["cloud-init-per", "once", "create_LV_CEPHVAR", "lvcreate", "-L", "60GB", "-n", "CEPHVAR", "-y", "metalvg0"],
+      ["cloud-init-per", "once", "create_LV_CONTAIN", "lvcreate", "-L", "60GB", "-n", "CONTAIN", "-y", "metalvg0"]
+    ]'
+
+    # cloud-init fs_setup for storage nodes
+    STORAGE_FILE_SYSTEMS='[
+      {"label": "CEPHETC", "filesystem": "ext4", "device": "/dev/disk/by-id/dm-name-metalvg0-CEPHETC", "partition": "auto", "overwrite": true},
+      {"label": "CEPHVAR", "filesystem": "ext4", "device": "/dev/disk/by-id/dm-name-metalvg0-CEPHVAR", "partition": "auto", "overwrite": true},
+      {"label": "CONTAIN", "filesystem": "xfs", "device": "/dev/disk/by-id/dm-name-metalvg0-CONTAIN", "partition": "auto", "overwrite": true}
+    ]'
+
+    # cloud-init mounts for storage nodes
+    STORAGE_MOUNTS='[
+      ["label": "CEPHETC", "mountpoint": "/etc/ceph", "filesystem": "auto", "options": "defaults"],
+      ["label": "CEPHVAR", "mountpoint": "/var/lib/ceph", "filesystem": "auto", "options": "defaults"],
+      ["label": "CONTAIN", "mountpoint": "/var/lib/containers", "filesystem": "auto", "options": "defaults"]
+    ]'
+
+    # Get xnames for all Management nodes
+    if IFS=$'\n' read -rd '' -a NCN_XNAMES; then
+      :
+    fi <<< "$(cray hsm state components list --role Management --type Node --format json | jq -r '.Components | map(.ID) | join("\n")')"
+
+    # Update BSS data for ncn-master nodes
+    for xname in "${NCN_XNAMES[@]}"; do
+      xname_bss="$(cray bss bootparameters list --format json --hosts "${xname}")"
+      jq --argjson master_bootcmd "$MASTER_BOOT_CMD" \
+         --argjson master_fs_setup "$MASTER_FILE_SYSTEMS" \
+         --argjson master_mounts "$MASTER_MOUNTS" \
+         --argjson worker_bootcmd "$WORKER_BOOT_CMD" \
+         --argjson worker_fs_setup "$WORKER_FILE_SYSTEMS" \
+         --argjson worker_mounts "$WORKER_MOUNTS" \
+         --argjson storage_bootcmd "$STORAGE_BOOT_CMD" \
+         --argjson storage_fs_setup "$STORAGE_FILE_SYSTEMS" \
+         --argjson storage_mounts "$STORAGE_MOUNTS" '
+         map(
+           if .["cloud-init"]["meta-data"]["shasta-role"] == "ncn-master" then
+             .["cloud-init"]["user-data"]["bootcmd"] = $master_bootcmd |
+             .["cloud-init"]["user-data"]["fs_setup"] = $masterfs_setup |
+             .["cloud-init"]["user-data"]["mounts"] = $mastermounts
+           elif .["cloud-init"]["meta-data"]["shasta-role"] == "ncn-worker" then
+             .["cloud-init"]["user-data"]["bootcmd"] = $worker_bootcmd |
+             .["cloud-init"]["user-data"]["fs_setup"] = $worker_fs_setup |
+             .["cloud-init"]["user-data"]["mounts"] = $worker_mounts
+           elif .["cloud-init"]["meta-data"]["shasta-role"] == "ncn-storage" then
+             .["cloud-init"]["user-data"]["bootcmd"] = $storage_bootcmd |
+             .["cloud-init"]["user-data"]["fs_setup"] = $storage_fs_setup |
+             .["cloud-init"]["user-data"]["mounts"] = $storage_mounts
+           else .
+         end)' <<< "$xname_bss" > "bss-patched-${xname}.json"
+
+      cray bss bootparameters update --file "bss-patched-${xname}.json" && rm "bss-patched-${xname}.json"
+    done
+
+  } >> "${LOG_FILE}" 2>&1
+  record_state "${state_name}" "$(hostname)" | tee -a "${LOG_FILE}"
+else
+  echo "====> ${state_name} has been completed" | tee -a "${LOG_FILE}"
+fi
+
 state_name="TDS_LOWER_CPU_REQUEST"
 state_recorded=$(is_state_recorded "${state_name}" "$(hostname)")
 if [[ ${state_recorded} == "0" && $(hostname) == "${PRIMARY_NODE}" && ${vshasta} == "false" ]]; then