Amd direct solver (#521)

* Working Ginkgo direct solver on AMD * fix build failure without ma57 * minor corrections * Update Ascent CI script to use ginkgo@ea106a945a390a1580baee4648c19ca2b665acdf * Add ginkgo_exec option to choose the hardware architecture the Ginkgo solver is run on. * Add tests for CUDA and HIP backends * Fix typo * Fix PNNL CI (#526) * Update marianas variables and add spack.yaml. * Add debugging lines for failing spack build. * Fix syntax error. * Update Newell variables and re-enable CI. * Fix bugs in newell variables. Fixup. * Disable ginkgo+cuda test on Marianas. * Bugfix test config on marianas. * Final attempt at disablingng specific tests. Co-authored-by: Nicholson Koukpaizan <koukpaizannk@ornl.gov> Co-authored-by: Cameron Rutherford <robert.rutherford@pnnl.gov>
LLNL · Aug 8, 2022 · 039f6e3 · 039f6e3
1 parent f131f57
commit 039f6e3
Show file tree

Hide file tree

Showing 13 changed files with 472 additions and 134 deletions.
diff --git a/.github/workflows/spack_build.yml b/.github/workflows/spack_build.yml
@@ -31,10 +31,11 @@ jobs:
         # for x86 systems. This allows us to use far more prebuilt packages,
         # which should speed up the builds by quite a bit.
         run: |
-          ls && pwd && \
-          . /opt/spack/share/spack/setup-env.sh && \
-          spack mirror add binary_mirror https://binaries.spack.io/develop && \
-          spack buildcache keys --install --trust && \
-          spack spec $SPACK_SPEC target=x86_64 && \
-          spack install --fail-fast $SPACK_SPEC target=x86_64
+          ls && pwd
+          . /opt/spack/share/spack/setup-env.sh
+          spack -d debug report
+          spack -d mirror add binary_mirror https://binaries.spack.io/develop
+          spack -d buildcache keys --install --trust
+          spack -d spec $SPACK_SPEC target=x86_64
+          spack -d --stacktrace install --fail-fast $SPACK_SPEC target=x86_64
 
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -115,6 +115,9 @@
         export SLURM_Q=`perl $WORKDIR/scripts/findIdleDLNodes.pl`
       fi
 
+      # Extra args for ctest
+      export CTEST_CMD=$CTEST_CMD
+
       sbatch -A EXASGD --exclusive -N 1 -n 8 -p $SLURM_Q -t $TIMELIMIT $SLURM_ARGS -o output -e output $WORKDIR/BUILD.sh $BUILD_SCRIPT_ARGS
       res=1
       set +xv
@@ -158,22 +161,24 @@ build_on_marianas:
     MY_CLUSTER: "marianas"
     TIMELIMIT: '1:30:00'
     SLURM_ARGS: --gres=gpu:1 --exclusive
+  # Through the steps, the argument to -E is automatically surrounded by quotes
+    CTEST_CMD: 'ctest -VV -E NlpSparse1_6|NlpSparse2_5'
   <<: *pnnl_tags_definition
   <<: *pnnl_script_definition
   rules:
     - if: '$CI_PROJECT_ROOT_NAMESPACE == "exasgd"'
 
-# build_on_newell:
-#   stage: default_build
-#   variables:
-#     SLURM_Q: "newell_shared"
-#     MY_CLUSTER: "newell"
-#     TIMELIMIT: '1:00:00'
-#     SLURM_ARGS: --gres=gpu:1 --exclusive
-#   <<: *pnnl_tags_definition
-#   <<: *pnnl_script_definition
-#   rules:
-#     - if: '$CI_PROJECT_ROOT_NAMESPACE == "exasgd"'
+build_on_newell:
+  stage: default_build
+  variables:
+    SLURM_Q: "newell_shared"
+    MY_CLUSTER: "newell"
+    TIMELIMIT: '1:00:00'
+    SLURM_ARGS: --gres=gpu:1 --exclusive
+  <<: *pnnl_tags_definition
+  <<: *pnnl_script_definition
+  rules:
+    - if: '$CI_PROJECT_ROOT_NAMESPACE == "exasgd"'
 
 build_on_incline:
   allow_failure: true

diff --git a/scripts/ascentVariables.sh b/scripts/ascentVariables.sh
@@ -22,7 +22,7 @@ module load exasgd-cub/1.16.0/gcc-9.1.0-o5zdbep
 # cuda@11.4.2%gcc@9.1.0~allow-unsupported-compilers~dev arch=linux-rhel8-power9le
 module load exasgd-cuda/11.4.2/gcc-9.1.0-4676kh5
 # ginkgo@glu%gcc@9.1.0+cuda~develtools~full_optimizations~hwloc~ipo~oneapi+openmp~rocm+shared build_type=Release cuda_arch=70 dev_path=/gpfs/wolf/proj-shared/csc359/src/ginkgo arch=linux-rhel8-power9le
-module load exasgd-ginkgo/glu/cuda-11.4.2/gcc-9.1.0-4ole5wn
+module load exasgd-ginkgo/glu/cuda-11.4.2/gcc-9.1.0-fpuykyc
 # gmp@6.2.1%gcc@9.1.0 libs=shared,static arch=linux-rhel8-power9le
 module load exasgd-gmp/6.2.1/gcc-9.1.0-umqilrg
 # gnuconfig@2021-08-14%gcc@9.1.0 arch=linux-rhel8-power9le

diff --git a/scripts/marianasVariables.sh b/scripts/marianasVariables.sh
@@ -1,68 +1,71 @@
 #  NOTE: The following is required when running from Gitlab CI via slurm job
 source /etc/profile.d/modules.sh
-module use -a /qfs/projects/exasgd/src/cameron-spack/share/spack/modules/linux-centos7-x86_64_v3
-
+module use -a /qfs/projects/exasgd/src/cameron/spack/share/spack/modules/linux-centos7-zen2
 
 # Load spack-built modules
-# autoconf@2.69%gcc@7.3.0 patches=35c4492,7793209,a49dd5b arch=linux-centos7-x86_64_v3
-module load autoconf-2.69-gcc-7.3.0-gvh7nxv
-# autoconf-archive@2022.02.11%gcc@7.3.0 patches=130cd48 arch=linux-centos7-x86_64_v3
-module load autoconf-archive-2022.02.11-gcc-7.3.0-lrajcp3
-# automake@1.16.5%gcc@7.3.0 arch=linux-centos7-x86_64_v3
-module load automake-1.16.5-gcc-7.3.0-la5kvuy
-# blt@0.4.1%gcc@7.3.0 arch=linux-centos7-x86_64_v3
-module load blt-0.4.1-gcc-7.3.0-qeolwyb
-# ca-certificates-mozilla@2022-03-29%gcc@7.3.0 arch=linux-centos7-x86_64_v3
-module load ca-certificates-mozilla-2022-03-29-gcc-7.3.0-fjb4zc5
-# camp@0.2.2%gcc@7.3.0+cuda~ipo~rocm~tests build_type=RelWithDebInfo cuda_arch=60 arch=linux-centos7-x86_64_v3
-module load camp-0.2.2-gcc-7.3.0-ifdwyok
-# cmake@3.23.2%gcc@7.3.0~doc+ncurses+ownlibs~qt build_type=Release arch=linux-centos7-x86_64_v3
-module load cmake-3.23.2-gcc-7.3.0-riu7fla
-# coinhsl@2015.06.23%gcc@7.3.0+blas arch=linux-centos7-x86_64_v3
-module load coinhsl-2015.06.23-gcc-7.3.0-r42slsl
-# cub@1.16.0%gcc@7.3.0 arch=linux-centos7-x86_64_v3
-module load cub-1.16.0-gcc-7.3.0-4zaltzb
-# ginkgo@glu%gcc@7.3.0+cuda~develtools~full_optimizations~hwloc~ipo~oneapi+openmp~rocm+shared build_type=Release cuda_arch=60 arch=linux-centos7-x86_64_v3
-module load ginkgo-glu-gcc-7.3.0-63ouzce
-# gmp@6.2.1%gcc@7.3.0 libs=shared,static arch=linux-centos7-x86_64_v3
-module load gmp-6.2.1-gcc-7.3.0-if7iflm
-# libsigsegv@2.13%gcc@7.3.0 arch=linux-centos7-x86_64_v3
-module load libsigsegv-2.13-gcc-7.3.0-n653jc7
-# libtool@2.4.7%gcc@7.3.0 arch=linux-centos7-x86_64_v3
-module load libtool-2.4.7-gcc-7.3.0-atzgxc2
-# m4@1.4.19%gcc@7.3.0+sigsegv patches=9dc5fbd,bfdffa7 arch=linux-centos7-x86_64_v3
-module load m4-1.4.19-gcc-7.3.0-lcthdqt
-# magma@2.6.2%gcc@7.3.0+cuda+fortran~ipo~rocm+shared build_type=RelWithDebInfo cuda_arch=60 arch=linux-centos7-x86_64_v3
-module load magma-2.6.2-gcc-7.3.0-kqvdxay
-# metis@5.1.0%gcc@7.3.0~gdb~int64~real64+shared build_type=Release patches=4991da9,b1225da arch=linux-centos7-x86_64_v3
-module load metis-5.1.0-gcc-7.3.0-xfajh3x
-# mpfr@4.1.0%gcc@7.3.0 libs=shared,static arch=linux-centos7-x86_64_v3
-module load mpfr-4.1.0-gcc-7.3.0-zcatq2v
-# ncurses@6.2%gcc@7.3.0~symlinks+termlib abi=none arch=linux-centos7-x86_64_v3
-module load ncurses-6.2-gcc-7.3.0-sqnhgdg
-# openblas@0.3.20%gcc@4.8.5~bignuma~consistent_fpcsr~ilp64+locking+pic+shared symbol_suffix=none threads=none arch=linux-centos7-x86_64_v3
-module load openblas-0.3.20-gcc-4.8.5-rpolrqa
-# openmpi@3.1.3%gcc@7.3.0~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java~legacylaunchers~lustre~memchecker~pmi+romio+rsh~singularity+static+vt+wrapper-rpath fabrics=none schedulers=none arch=linux-centos7-x86_64_v3
-module load openmpi-3.1.3-gcc-7.3.0-2ocdjy4
-# openssl@1.1.1p%gcc@7.3.0~docs~shared certs=mozilla arch=linux-centos7-x86_64_v3
-module load openssl-1.1.1p-gcc-7.3.0-tz3ln5w
-# perl@5.26.0%gcc@7.3.0+cpanm+shared+threads patches=0eac10e,8cf4302 arch=linux-centos7-x86_64_v3
-module load perl-5.26.0-gcc-7.3.0-f7w3oxq
-# pkgconf@1.8.0%gcc@7.3.0 arch=linux-centos7-x86_64_v3
-module load pkgconf-1.8.0-gcc-7.3.0-gxowfdy
-# raja@0.14.0%gcc@7.3.0+cuda~examples~exercises~ipo+openmp~rocm+shared~tests build_type=RelWithDebInfo cuda_arch=60 arch=linux-centos7-x86_64_v3
-module load raja-0.14.0-gcc-7.3.0-vtbmo6k
-# suite-sparse@5.10.1%gcc@7.3.0~cuda~graphblas~openmp+pic~tbb arch=linux-centos7-x86_64_v3
-module load suite-sparse-5.10.1-gcc-7.3.0-thhoxwy
-# texinfo@6.5%gcc@7.3.0 patches=12f6edb,1732115 arch=linux-centos7-x86_64_v3
-module load texinfo-6.5-gcc-7.3.0-crm3bgr
-# umpire@6.0.0%gcc@7.3.0+c+cuda~device_alloc~deviceconst~examples~fortran~ipo~numa~openmp~rocm~shared build_type=RelWithDebInfo cuda_arch=60 tests=none arch=linux-centos7-x86_64_v3
-module load umpire-6.0.0-gcc-7.3.0-z22n3zy
-# zlib@1.2.12%gcc@7.3.0+optimize+pic+shared patches=0d38234 arch=linux-centos7-x86_64_v3
-module load zlib-1.2.12-gcc-7.3.0-hq7ha7b
+# autoconf@2.69%gcc@10.2.0 patches=35c4492,7793209,a49dd5b arch=linux-centos7-zen2
+module load autoconf-2.69-gcc-10.2.0-r677m42
+# autoconf-archive@2022.02.11%gcc@10.2.0 patches=130cd48 arch=linux-centos7-zen2
+module load autoconf-archive-2022.02.11-gcc-10.2.0-pbrbzut
+# automake@1.16.5%gcc@10.2.0 arch=linux-centos7-zen2
+module load automake-1.16.5-gcc-10.2.0-j4bwm4o
+# blt@0.4.1%gcc@10.2.0 arch=linux-centos7-zen2
+module load blt-0.4.1-gcc-10.2.0-tanugdw
+# ca-certificates-mozilla@2022-07-19%gcc@10.2.0 arch=linux-centos7-zen2
+module load ca-certificates-mozilla-2022-07-19-gcc-10.2.0-h2opehw
+# camp@0.2.3%gcc@10.2.0+cuda~ipo~rocm~tests build_type=RelWithDebInfo cuda_arch=60 arch=linux-centos7-zen2
+module load camp-0.2.3-gcc-10.2.0-vpkkybx
+# cmake@3.23.2%gcc@10.2.0~doc+ncurses+ownlibs~qt build_type=Release arch=linux-centos7-zen2
+module load cmake-3.23.2-gcc-10.2.0-i24avzq
+# coinhsl@2019.05.21%gcc@10.2.0+blas arch=linux-centos7-zen2
+module load coinhsl-2019.05.21-gcc-10.2.0-j7hsujd
+# cub@1.16.0%gcc@10.2.0 arch=linux-centos7-zen2
+module load cub-1.16.0-gcc-10.2.0-ovgrtom
+# diffutils@3.8%gcc@10.2.0 arch=linux-centos7-zen2
+module load diffutils-3.8-gcc-10.2.0-mjfwces
+# ginkgo@glu_experimental%gcc@10.2.0+cuda~develtools~full_optimizations~hwloc~ipo~oneapi+openmp~rocm+shared build_type=Release cuda_arch=60 arch=linux-centos7-zen2
+module load ginkgo-glu_experimental-gcc-10.2.0-dbmokiq
+# gmp@6.2.1%gcc@10.2.0 libs=shared,static arch=linux-centos7-zen2
+module load gmp-6.2.1-gcc-10.2.0-ac4z3oa
+# libiconv@1.16%gcc@10.2.0 libs=shared,static arch=linux-centos7-zen2
+module load libiconv-1.16-gcc-10.2.0-gbg7l5p
+# libsigsegv@2.13%gcc@10.2.0 arch=linux-centos7-zen2
+module load libsigsegv-2.13-gcc-10.2.0-aj5goyi
+# libtool@2.4.7%gcc@10.2.0 arch=linux-centos7-zen2
+module load libtool-2.4.7-gcc-10.2.0-mzc2mvw
+# m4@1.4.19%gcc@10.2.0+sigsegv patches=9dc5fbd,bfdffa7 arch=linux-centos7-zen2
+module load m4-1.4.19-gcc-10.2.0-k5kkyx6
+# magma@2.6.2%gcc@10.2.0+cuda+fortran~ipo~rocm+shared build_type=RelWithDebInfocuda_arch=60 arch=linux-centos7-zen2
+module load magma-2.6.2-gcc-10.2.0-o7gg2nj
+# metis@5.1.0%gcc@10.2.0~gdb~int64~real64+shared build_type=Release patches=4991da9,b1225da arch=linux-centos7-zen2
+module load metis-5.1.0-gcc-10.2.0-h2r63pj
+# mpfr@4.1.0%gcc@10.2.0 libs=shared,static arch=linux-centos7-zen2
+module load mpfr-4.1.0-gcc-10.2.0-ixeo4lu
+# ncurses@6.2%gcc@10.2.0~symlinks+termlib abi=none arch=linux-centos7-zen2
+module load ncurses-6.2-gcc-10.2.0-3b2uqgl
+# openblas@0.3.20%gcc@10.2.0~bignuma~consistent_fpcsr~ilp64+locking+pic+shared symbol_suffix=none threads=none arch=linux-centos7-zen2
+module load openblas-0.3.20-gcc-10.2.0-qhcutll
+# openmpi@4.1.0%gcc@10.2.0~atomics~cuda~cxx~cxx_exceptions~gpfs~internal-hwloc~java~legacylaunchers~lustre~memchecker+romio+rsh~singularity+static+vt+wrapper-rpath fabrics=none patches=60ce20b schedulers=none arch=linux-centos7-zen2
+module load openmpi-4.1.0-gcc-10.2.0-wnndpcg
+# openssl@1.1.1q%gcc@10.2.0~docs~shared certs=mozilla patches=3fdcf2d arch=linux-centos7-zen2
+module load openssl-1.1.1q-gcc-10.2.0-t5hsb3s
+# perl@5.26.0%gcc@10.2.0+cpanm+shared+threads patches=0eac10e,8cf4302 arch=linux-centos7-zen2
+module load perl-5.26.0-gcc-10.2.0-l2yiybo
+# pkgconf@1.8.0%gcc@10.2.0 arch=linux-centos7-zen2
+module load pkgconf-1.8.0-gcc-10.2.0-fuflwbl
+# raja@0.14.0%gcc@10.2.0+cuda~examples~exercises~ipo+openmp~rocm+shared~tests build_type=RelWithDebInfo cuda_arch=60 arch=linux-centos7-zen2
+module load raja-0.14.0-gcc-10.2.0-pc2ckhw
+# suite-sparse@5.10.1%gcc@10.2.0~cuda~graphblas~openmp+pic~tbb arch=linux-centos7-zen2
+module load suite-sparse-5.10.1-gcc-10.2.0-jkighdn
+# texinfo@6.5%gcc@10.2.0 patches=12f6edb,1732115 arch=linux-centos7-zen2
+module load texinfo-6.5-gcc-10.2.0-mzqgqla
+# umpire@6.0.0%gcc@10.2.0+c+cuda~device_alloc~deviceconst~examples~fortran~ipo~numa~openmp~rocm~shared build_type=RelWithDebInfo cuda_arch=60 tests=none arch=linux-centos7-zen2
+module load umpire-6.0.0-gcc-10.2.0-eunwzka
+# zlib@1.2.12%gcc@10.2.0+optimize+pic+shared patches=0d38234 arch=linux-centos7-zen2
+module load zlib-1.2.12-gcc-10.2.0-gnkqokp
 
 # Load system modules
-module load gcc/7.3.0
+module load gcc/10.2.0
 module load cuda/11.4
 
 [ -f $PWD/nvblas.conf ] && rm $PWD/nvblas.conf

diff --git a/scripts/newellVariables.sh b/scripts/newellVariables.sh
@@ -1,50 +1,87 @@
 #  NOTE: The following is required when running from Gitlab CI via slurm job
 source /etc/profile.d/modules.sh
+module purge
 module use -a /usr/share/Modules/modulefiles
 module use -a /share/apps/modules/tools
 module use -a /share/apps/modules/compilers
 module use -a /share/apps/modules/mpi
 module use -a /etc/modulefiles
-module use -a /qfs/projects/exasgd/src/cameron-spack/share/spack/modules/linux-rhel7-power9le
+module use -a /qfs/projects/exasgd/src/cameron/spack/share/spack/modules/linux-centos8-power9le/
 
 # Load spack-built modules
-# blt@0.4.1%gcc@7.4.0 arch=linux-rhel7-power9le
-module load blt-0.4.1-gcc-7.4.0-2th7jgq
-# camp@0.2.2%gcc@7.4.0+cuda~ipo~rocm~tests build_type=RelWithDebInfo cuda_arch=70 arch=linux-rhel7-power9le
-module load camp-0.2.2-gcc-7.4.0-vsu2jwh
-# cmake@3.23.1%gcc@7.4.0~doc+ncurses+ownlibs~qt build_type=Release arch=linux-rhel7-power9le
-module load cmake-3.23.1-gcc-7.4.0-ckfugtf
-# coinhsl@2015.06.23%gcc@7.4.0+blas arch=linux-rhel7-power9le
-module load coinhsl-2015.06.23-gcc-7.4.0-ts5vjfq
-# cub@1.12.0%gcc@7.4.0 arch=linux-rhel7-power9le
-module load cub-1.12.0-gcc-7.4.0-4qyvoqn
-# ginkgo@glu%gcc@7.4.0+cuda~develtools~full_optimizations~hwloc~ipo~oneapi+openmp~rocm+shared build_type=Release cuda_arch=70 arch=linux-rhel7-power9le
-module load ginkgo-glu-gcc-7.4.0-r5wjmju
-# gmp@6.2.1%gcc@7.4.0 libs=shared,static arch=linux-rhel7-power9le
-module load gmp-6.2.1-gcc-7.4.0-oea2aet
-# magma@2.6.2%gcc@7.4.0+cuda+fortran~ipo~rocm+shared build_type=RelWithDebInfo cuda_arch=70 arch=linux-rhel7-power9le
-module load magma-2.6.2-gcc-7.4.0-6yuqfpm
-# metis@5.1.0%gcc@7.4.0~gdb~int64~real64+shared build_type=Release patches=4991da9,b1225da arch=linux-rhel7-power9le
-module load metis-5.1.0-gcc-7.4.0-shhhyku
-# mpfr@4.1.0%gcc@7.4.0 libs=shared,static arch=linux-rhel7-power9le
-module load mpfr-4.1.0-gcc-7.4.0-tz5esun
-# ncurses@6.2%gcc@7.4.0~symlinks+termlib abi=none arch=linux-rhel7-power9le
-module load ncurses-6.2-gcc-7.4.0-kqhmmpv
-# openblas@0.3.20%gcc@7.4.0~bignuma~consistent_fpcsr~ilp64+locking+pic+shared symbol_suffix=none threads=none arch=linux-rhel7-power9le
-module load openblas-0.3.20-gcc-7.4.0-3zdqw2i
-# raja@0.14.0%gcc@7.4.0+cuda~examples~exercises~ipo+openmp~rocm+shared~tests build_type=RelWithDebInfo cuda_arch=70 arch=linux-rhel7-power9le
-module load raja-0.14.0-gcc-7.4.0-sew5thv
-# suite-sparse@5.10.1%gcc@7.4.0~cuda~graphblas~openmp+pic~tbb arch=linux-rhel7-power9le
-module load suite-sparse-5.10.1-gcc-7.4.0-e5qockg
-# umpire@6.0.0%gcc@7.4.0+c+cuda~deviceconst~examples~fortran~ipo~numa~openmp~rocm~shared build_type=RelWithDebInfo cuda_arch=70 tests=none arch=linux-rhel7-power9le
-module load umpire-6.0.0-gcc-7.4.0-rpwrj4p
-# zlib@1.2.12%gcc@7.4.0+optimize+pic+shared patches=0d38234 arch=linux-rhel7-power9le
-module load zlib-1.2.12-gcc-7.4.0-d6xlzc6
+# autoconf@2.69%gcc@8.5.0 patches=35c4492,7793209,a49dd5b arch=linux-centos8-power9le
+module load autoconf-2.69-gcc-8.5.0-2mzbyqj
+# autoconf-archive@2022.02.11%gcc@8.5.0 patches=130cd48 arch=linux-centos8-power9le
+module load autoconf-archive-2022.02.11-gcc-8.5.0-nolgalj
+# automake@1.16.5%gcc@8.5.0 arch=linux-centos8-power9le
+module load automake-1.16.5-gcc-8.5.0-pnnvoal
+# berkeley-db@18.1.40%gcc@8.5.0+cxx~docs+stl patches=b231fcc arch=linux-centos8-power9le
+module load berkeley-db-18.1.40-gcc-8.5.0-cuzn6qn
+# blt@0.4.1%gcc@8.5.0 arch=linux-centos8-power9le
+module load blt-0.4.1-gcc-8.5.0-4drqwl4
+# bzip2@1.0.8%gcc@8.5.0~debug~pic+shared arch=linux-centos8-power9le
+module load bzip2-1.0.8-gcc-8.5.0-tsweuon
+# ca-certificates-mozilla@2022-07-19%gcc@8.5.0 arch=linux-centos8-power9le
+module load ca-certificates-mozilla-2022-07-19-gcc-8.5.0-db3wqwx
+# camp@0.2.3%gcc@8.5.0+cuda~ipo~rocm~tests build_type=RelWithDebInfo cuda_arch=70 arch=linux-centos8-power9le
+module load camp-0.2.3-gcc-8.5.0-mtks7g5
+# cmake@3.23.2%gcc@8.5.0~doc+ncurses+ownlibs~qt build_type=Release arch=linux-centos8-power9le
+module load cmake-3.23.2-gcc-8.5.0-tpplkft
+# coinhsl@2019.05.21%gcc@8.5.0+blas arch=linux-centos8-power9le
+module load coinhsl-2019.05.21-gcc-8.5.0-vng3am5
+# cub@1.16.0%gcc@8.5.0 arch=linux-centos8-power9le
+module load cub-1.16.0-gcc-8.5.0-p3cnthb
+# diffutils@3.8%gcc@8.5.0 arch=linux-centos8-power9le
+module load diffutils-3.8-gcc-8.5.0-ppyuisg
+# gdbm@1.19%gcc@8.5.0 arch=linux-centos8-power9le
+module load gdbm-1.19-gcc-8.5.0-unfo3x4
+# ginkgo@glu_experimental%gcc@8.5.0+cuda~develtools~full_optimizations~hwloc~ipo~oneapi+openmp~rocm+shared build_type=Release cuda_arch=70 arch=linux-centos8-power9le
+module load ginkgo-glu_experimental-gcc-8.5.0-m3p5yj4
+# gmp@6.2.1%gcc@8.5.0 libs=shared,static arch=linux-centos8-power9le
+module load gmp-6.2.1-gcc-8.5.0-xlcuuht
+# gnuconfig@2021-08-14%gcc@8.5.0 arch=linux-centos8-power9le
+module load gnuconfig-2021-08-14-gcc-8.5.0-qjyg7ls
+# hiop@develop%gcc@8.5.0+cuda+cusolver+deepchecking+ginkgo~ipo~jsrun+kron+mpi+raja~rocm~shared+sparse build_type=RelWithDebInfo cuda_arch=70 dev_path=/qfs/projects/exasgd/src/cameron/hiop-git arch=linux-centos8-power9le
+module load hiop-develop-gcc-8.5.0-p2l3auf
+# libiconv@1.16%gcc@8.5.0 libs=shared,static arch=linux-centos8-power9le
+module load libiconv-1.16-gcc-8.5.0-qqwmnok
+# libsigsegv@2.13%gcc@8.5.0 arch=linux-centos8-power9le
+module load libsigsegv-2.13-gcc-8.5.0-pa77xit
+# libtool@2.4.7%gcc@8.5.0 arch=linux-centos8-power9le
+module load libtool-2.4.7-gcc-8.5.0-kxdso3c
+# m4@1.4.19%gcc@8.5.0+sigsegv patches=9dc5fbd,bfdffa7 arch=linux-centos8-power9le
+module load m4-1.4.19-gcc-8.5.0-untfsqf
+# magma@2.6.2%gcc@8.5.0+cuda+fortran~ipo~rocm+shared build_type=RelWithDebInfo cuda_arch=70 arch=linux-centos8-power9le
+module load magma-2.6.2-gcc-8.5.0-4oanlpm
+# metis@5.1.0%gcc@8.5.0~gdb~int64~real64+shared build_type=Release patches=4991da9,b1225da arch=linux-centos8-power9le
+module load metis-5.1.0-gcc-8.5.0-hcv2jnr
+# mpfr@4.1.0%gcc@8.5.0 libs=shared,static arch=linux-centos8-power9le
+module load mpfr-4.1.0-gcc-8.5.0-esdxmf2
+# ncurses@6.2%gcc@8.5.0~symlinks+termlib abi=none arch=linux-centos8-power9le
+module load ncurses-6.2-gcc-8.5.0-v24hmxo
+# openblas@0.3.20%gcc@8.5.0~bignuma~consistent_fpcsr~ilp64+locking+pic+shared symbol_suffix=none threads=none arch=linux-centos8-power9le
+module load openblas-0.3.20-gcc-8.5.0-rwstn2s
+# perl@5.34.1%gcc@8.5.0+cpanm+shared+threads arch=linux-centos8-power9le
+module load perl-5.34.1-gcc-8.5.0-fn534xj
+# pkgconf@1.8.0%gcc@8.5.0 arch=linux-centos8-power9le
+module load pkgconf-1.8.0-gcc-8.5.0-imrnro2
+# raja@0.14.0%gcc@8.5.0+cuda+examples+exercises~ipo+openmp~rocm+shared~tests build_type=RelWithDebInfo cuda_arch=70 arch=linux-centos8-power9le
+module load raja-0.14.0-gcc-8.5.0-yd3im6p
+# readline@8.1.2%gcc@8.5.0 arch=linux-centos8-power9le
+module load readline-8.1.2-gcc-8.5.0-l4hzlyf
+# suite-sparse@5.10.1%gcc@8.5.0~cuda~graphblas~openmp+pic~tbb arch=linux-centos8-power9le
+module load suite-sparse-5.10.1-gcc-8.5.0-6ra3sp4
+# texinfo@6.5%gcc@8.5.0 patches=12f6edb,1732115 arch=linux-centos8-power9le
+module load texinfo-6.5-gcc-8.5.0-fvxyl2q
+# umpire@6.0.0%gcc@8.5.0+c+cuda~device_alloc~deviceconst+examples~fortran~ipo~numa~openmp~rocm~shared build_type=RelWithDebInfo cuda_arch=70 tests=none arch=linux-centos8-power9le
+module load umpire-6.0.0-gcc-8.5.0-ogbxb44
+# zlib@1.2.12%gcc@8.5.0+optimize+pic+shared patches=0d38234 arch=linux-centos8-power9le
+module load zlib-1.2.12-gcc-8.5.0-spb5k73
 
 # Load system modules
-module load gcc/7.4.0
-module load cuda/10.2
-module load openmpi/3.1.5
+module load gcc/8.5.0
+module load cuda/11.4
+module load openmpi/4.1.4
 
 [ -f $PWD/nvblas.conf ] && rm $PWD/nvblas.conf
 cat > $PWD/nvblas.conf <<-EOD