Upgrade components and install PMIx

alma/alma-8.x/alma-8.7-hpc/install.sh

@@ -19,6 +19,9 @@ $ALMA_COMMON_DIR/install_lustre_client.sh "8"
 # install mellanox ofed
 ./install_mellanoxofed.sh
 
+# install PMIX
+$ALMA_COMMON_DIR/../alma-8.x/common/install_pmix.sh
+
 # install mpi libraries
 ./install_mpis.sh
 
@@ -32,10 +35,10 @@ $ALMA_COMMON_DIR/install_lustre_client.sh "8"
 ./install_intel_libs.sh
 
 # cleanup downloaded tarballs - clear some space
-rm -rf *.tgz *.bz2 *.tbz *.tar.gz *.run *.deb *_offline.sh
-rm -rf /tmp/MLNX_OFED_LINUX* /tmp/*conf*
-rm -rf /var/intel/ /var/cache/*
-rm -Rf -- */
+#rm -rf *.tgz *.bz2 *.tbz *.tar.gz *.run *.deb *_offline.sh
+#rm -rf /tmp/MLNX_OFED_LINUX* /tmp/*conf*
+#rm -rf /var/intel/ /var/cache/*
+#rm -Rf -- */
 
 # Install NCCL
 ./install_nccl.sh

alma/alma-8.x/alma-8.7-hpc/install_mpis.sh

@@ -19,12 +19,20 @@ HPCX_DOWNLOAD_URL=$(jq -r '.url' <<< $hpcx_metadata)
 TARBALL=$(basename $HPCX_DOWNLOAD_URL)
 HPCX_FOLDER=$(basename $HPCX_DOWNLOAD_URL .tbz)
 
+PMIX_VERSION=$(jq -r '.pmix."'"$DISTRIBUTION"'".version' <<< $COMPONENT_VERSIONS)
+PMIX_PATH=${INSTALL_PREFIX}/pmix/${PMIX_VERSION:0:-2}
+
 $COMMON_DIR/download_and_verify.sh $HPCX_DOWNLOAD_URL $HPCX_SHA256
 tar -xvf ${TARBALL}
+
+sed -i "s/\/build-result\//\/opt\//" ${HPCX_FOLDER}/hcoll/lib/pkgconfig/hcoll.pc
 mv ${HPCX_FOLDER} ${INSTALL_PREFIX}
 HPCX_PATH=${INSTALL_PREFIX}/${HPCX_FOLDER}
 $COMMON_DIR/write_component_version.sh "HPCX" $HPCX_VERSION
 
+# rebuild HPCX with PMIx
+${HPCX_PATH}/utils/hpcx_rebuild.sh --with-hcoll --ompi-extra-config --with-pmix=${PMIX_PATH}
+
 # exclude ucx from updates
 sed -i "$ s/$/ ucx*/" /etc/dnf/dnf.conf
 
@@ -38,7 +46,7 @@ cat << EOF >> /usr/share/Modules/modulefiles/mpi/hpcx-${HPCX_VERSION}
 #  HPCx ${HPCX_VERSION}
 #
 conflict        mpi
-module load ${HPCX_PATH}/modulefiles/hpcx
+module load ${HPCX_PATH}/modulefiles/hpcx-rebuild
 EOF
 
 # Create symlinks for modulefiles

alma/alma-8.x/common/install_nvidiagpudriver.sh

@@ -72,6 +72,10 @@ yum install -y ./nvidia-fabric-manager-${NVIDIA_FABRICMANAGER_VERSION}.x86_64.rp
 sed -i "$ s/$/ nvidia-fabric-manager/" /etc/dnf/dnf.conf
 $COMMON_DIR/write_component_version.sh "NVIDIA_FABRIC_MANAGER" ${NVIDIA_FABRICMANAGER_VERSION}
 
+systemctl enable nvidia-fabricmanager
+systemctl start nvidia-fabricmanager
+systemctl is-active --quiet nvidia-fabricmanager
+
 # cleanup downloaded files
 rm -rf *.run *tar.gz *.rpm
 rm -rf -- */

alma/alma-8.x/common/install_pmix.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -ex
+
+PMIX_VERSION=$(jq -r '.pmix."'"$DISTRIBUTION"'".version' <<< $COMPONENT_VERSIONS)
+REPO_DIR="$ALMA_COMMON_DIR/../alma-8.x/common"
+
+cp ${REPO_DIR}/slurmel8.repo /etc/yum.repos.d/slurm.repo
+
+## This package is pre-installed in all hpc images used by cyclecloud, but if customer wants to
+## build an image from generic marketplace images then this package sets up the right gpg keys for PMC.
+if [ ! -e /etc/yum.repos.d/microsoft-prod.repo ];then
+   curl -sSL -O https://packages.microsoft.com/config/rhel/8/packages-microsoft-prod.rpm
+   rpm -i packages-microsoft-prod.rpm
+   rm packages-microsoft-prod.rpm
+fi
+
+dnf config-manager --set-enabled powertools
+yum -y install pmix-$PMIX_VERSION.el8 hwloc-devel libevent-devel munge-devel
+
+$COMMON_DIR/write_component_version.sh "PMIX" ${PMIX_VERSION}

alma/alma-8.x/common/slurmel8.repo

@@ -0,0 +1,7 @@
+[slurm]
+name=Slurm Workload Manager
+baseurl=https://packages.microsoft.com/yumrepos/slurm-el8-insiders
+enabled=1
+gpgcheck=1
+gpgkey=https://packages.microsoft.com/keys/microsoft.asc
+priority=10

alma/common/install_mpis.sh

@@ -1,12 +1,15 @@
 #!/bin/bash
-set -e
+set -ex
 
 GCC_VERSION=$1
 HPCX_PATH=$2
 
+PMIX_VERSION=$(jq -r '.pmix."'"$DISTRIBUTION"'".version' <<< $COMPONENT_VERSIONS)
+
 HCOLL_PATH=${HPCX_PATH}/hcoll
 UCX_PATH=${HPCX_PATH}/ucx
 INSTALL_PREFIX=/opt
+PMIX_PATH=${INSTALL_PREFIX}/pmix/${PMIX_VERSION:0:-2}
 
 # Load gcc
 export PATH=/opt/${GCC_VERSION}/bin:$PATH
@@ -41,7 +44,9 @@ OMPI_FOLDER=$(basename $OMPI_DOWNLOAD_URL .tar.gz)
 $COMMON_DIR/download_and_verify.sh $OMPI_DOWNLOAD_URL $OMPI_SHA256
 tar -xvf $TARBALL
 cd $OMPI_FOLDER
-./configure --prefix=${INSTALL_PREFIX}/openmpi-${OMPI_VERSION} --with-ucx=${UCX_PATH} --with-hcoll=${HCOLL_PATH} --enable-mpirun-prefix-by-default --with-platform=contrib/platform/mellanox/optimized && make -j$(nproc) && make install
+./configure --prefix=${INSTALL_PREFIX}/openmpi-${OMPI_VERSION} --with-ucx=${UCX_PATH} --with-hcoll=${HCOLL_PATH} --with-pmix=${PMIX_PATH} --enable-mpirun-prefix-by-default --with-platform=contrib/platform/mellanox/optimized
+make -j$(nproc) 
+make install
 cd ..
 $COMMON_DIR/write_component_version.sh "OMPI" ${OMPI_VERSION}
 
@@ -57,7 +62,9 @@ IMPI_OFFLINE_INSTALLER=$(basename $IMPI_DOWNLOAD_URL)
 
 $COMMON_DIR/download_and_verify.sh $IMPI_DOWNLOAD_URL $IMPI_SHA256
 bash $IMPI_OFFLINE_INSTALLER -s -a -s --eula accept
-mv ${INSTALL_PREFIX}/intel/oneapi/mpi/${IMPI_VERSION}/modulefiles/mpi ${INSTALL_PREFIX}/intel/oneapi/mpi/${IMPI_VERSION}/modulefiles/impi
+
+impi_2021_version=${IMPI_VERSION:0:-2}
+mv ${INSTALL_PREFIX}/intel/oneapi/mpi/${impi_2021_version}/etc/modulefiles/mpi ${INSTALL_PREFIX}/intel/oneapi/mpi/${impi_2021_version}/etc/modulefiles/impi
 $COMMON_DIR/write_component_version.sh "IMPI" ${IMPI_VERSION}
 
 # Setup module files for MPIs
@@ -100,25 +107,25 @@ setenv          MPI_HOME        /opt/openmpi-${OMPI_VERSION}
 EOF
 
 #IntelMPI-v2021
-cat << EOF >> /usr/share/Modules/modulefiles/mpi/impi_${IMPI_VERSION}
+cat << EOF >> /usr/share/Modules/modulefiles/mpi/impi_${impi_2021_version}
 #%Module 1.0
 #
-#  Intel MPI ${IMPI_VERSION}
+#  Intel MPI ${impi_2021_version}
 #
 conflict        mpi
-module load /opt/intel/oneapi/mpi/${IMPI_VERSION}/modulefiles/impi
-setenv          MPI_BIN         /opt/intel/oneapi/mpi/${IMPI_VERSION}/bin
-setenv          MPI_INCLUDE     /opt/intel/oneapi/mpi/${IMPI_VERSION}/include
-setenv          MPI_LIB         /opt/intel/oneapi/mpi/${IMPI_VERSION}/lib
-setenv          MPI_MAN         /opt/intel/oneapi/mpi/${IMPI_VERSION}/man
-setenv          MPI_HOME        /opt/intel/oneapi/mpi/${IMPI_VERSION}
+module load /opt/intel/oneapi/mpi/${impi_2021_version}/etc/modulefiles/impi/${impi_2021_version}
+setenv          MPI_BIN         /opt/intel/oneapi/mpi/${impi_2021_version}/bin
+setenv          MPI_INCLUDE     /opt/intel/oneapi/mpi/${impi_2021_version}/include
+setenv          MPI_LIB         /opt/intel/oneapi/mpi/${impi_2021_version}/lib
+setenv          MPI_MAN         /opt/intel/oneapi/mpi/${impi_2021_version}/share/man
+setenv          MPI_HOME        /opt/intel/oneapi/mpi/${impi_2021_version}
 EOF
 
 # Create symlinks for modulefiles
 ln -s /usr/share/Modules/modulefiles/mpi/mvapich2-${MVAPICH2_VERSION} /usr/share/Modules/modulefiles/mpi/mvapich2
 ln -s /usr/share/Modules/modulefiles/mpi/openmpi-${OMPI_VERSION} /usr/share/Modules/modulefiles/mpi/openmpi
-ln -s /usr/share/Modules/modulefiles/mpi/impi_${IMPI_VERSION} /usr/share/Modules/modulefiles/mpi/impi-2021
+ln -s /usr/share/Modules/modulefiles/mpi/impi_${impi_2021_version} /usr/share/Modules/modulefiles/mpi/impi-2021
 
 # cleanup downloaded tarballs and other installation files/folders
-rm -rf *.tar.gz *offline.sh
-rm -rf -- */
+# rm -rf *.tar.gz *offline.sh
+# rm -rf -- */

customizations/ndv4.sh

@@ -15,8 +15,9 @@ EOF
 
 ## NVIDIA Fabric manager
 systemctl enable nvidia-fabricmanager
-systemctl start nvidia-fabricmanager
-systemctl is-active --quiet nvidia-fabricmanager
+# systemctl stop nvidia-fabricmanager
+# systemctl start nvidia-fabricmanager
+# systemctl is-active --quiet nvidia-fabricmanager
 
 error_code=$?
 if [ ${error_code} -ne 0 ]

customizations/ndv5.sh

@@ -15,8 +15,9 @@ EOF
 
 ## NVIDIA Fabric manager
 systemctl enable nvidia-fabricmanager
-systemctl start nvidia-fabricmanager
-systemctl is-active --quiet nvidia-fabricmanager
+# systemctl stop nvidia-fabricmanager
+# systemctl start nvidia-fabricmanager
+# systemctl is-active --quiet nvidia-fabricmanager
 
 error_code=$?
 if [ ${error_code} -ne 0 ]