From f3e5fe3f5784d42fb0108f523753412b715a2aa7 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 14 Feb 2023 16:33:51 -0700 Subject: [PATCH 01/28] Update Narwhal site config with GNU compiler --- configs/sites/narwhal/compilers.yaml | 18 ++++++++++++++++++ configs/sites/narwhal/packages.yaml | 5 ++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/configs/sites/narwhal/compilers.yaml b/configs/sites/narwhal/compilers.yaml index 183f65bb1..575adbab1 100644 --- a/configs/sites/narwhal/compilers.yaml +++ b/configs/sites/narwhal/compilers.yaml @@ -20,3 +20,21 @@ compilers:: set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] + - compiler: gcc@10.3.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + target: any + modules: + - PrgEnv-gnu/8.3.2 + - gcc/10.3.0 + environment: + prepend_path: + LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.08.1.1/INTEL/19.0/x86_64/lib' + set: + CRAYPE_LINK_TYPE: 'dynamic' + extra_rpaths: [] diff --git a/configs/sites/narwhal/packages.yaml b/configs/sites/narwhal/packages.yaml index c722fba5b..bf015cb12 100644 --- a/configs/sites/narwhal/packages.yaml +++ b/configs/sites/narwhal/packages.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.3.0] + compiler:: [intel@2021.3.0, gcc@10.3.0] providers: mpi:: [cray-mpich@8.1.14] @@ -12,6 +12,9 @@ packages: - spec: cray-mpich@8.1.14%intel@2021.3.0 ~wrappers modules: - cray-mpich/8.1.14 + - spec: cray-mpich@8.1.14%gcc@10.3.0 ~wrappers + modules: + - cray-mpich/8.1.14 #intel-oneapi-tbb: # externals: # - spec: intel-oneapi-tbb@2021.3.0 From 021c6f72acb1953c34119740ee29e5c72b433624 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 14 Feb 2023 17:14:25 -0700 Subject: [PATCH 02/28] Bug fix in configs/sites/narwhal/compilers.yaml --- configs/sites/narwhal/compilers.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configs/sites/narwhal/compilers.yaml b/configs/sites/narwhal/compilers.yaml index 575adbab1..f78e2c987 100644 --- a/configs/sites/narwhal/compilers.yaml +++ b/configs/sites/narwhal/compilers.yaml @@ -20,7 +20,8 @@ compilers:: set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] - - compiler: gcc@10.3.0 + - compiler: + spec: gcc@10.3.0 paths: cc: cc cxx: CC From b26379bfff7f7dfd9d37f83158b04dfe8d3bcf13 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 15 Feb 2023 09:18:02 -0700 Subject: [PATCH 03/28] Update documentation for narwhal --- doc/source/Platforms.rst | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/doc/source/Platforms.rst b/doc/source/Platforms.rst index 40ca4739e..7a9a877ed 100644 --- a/doc/source/Platforms.rst +++ b/doc/source/Platforms.rst @@ -31,7 +31,9 @@ spack-stack-v1 +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ | NASA Discover GNU | Dom Heinzeller | ``/discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-10.1.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ -| NAVY HPCMP Narwhal | Dom Heinzeller | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.4.0/install`` | +| NAVY HPCMP Narwhal Intel | Dom Heinzeller | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.4.0/install`` | ++----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ +| NAVY HPCMP Narwhal GNU (**TEMPORARY**) | Dom Heinzeller | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-test-20230214/envs/skylab-dev-gnu-10.3.0/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ | NCAR-Wyoming Casper | Dom Heinzeller | ``/glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-19.1.1.217-casper/install`` | +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+ @@ -144,7 +146,7 @@ For ``spack-stack-1.2.0``/``skylab-3.0.0`` with GNU, load the following modules NAVY HPCMP Narwhal ------------------------------ -The following is required for building new spack environments and for using spack to build and run software. +With Intel, the following is required for building new spack environments and for using spack to build and run software: .. code-block:: console @@ -171,6 +173,35 @@ For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following module module load stack-cray-mpich/8.1.14 module load stack-python/3.9.7 +With GNU, the following is required for building new spack environments and for using spack to build and run software: + +.. code-block:: console + + module unload PrgEnv-cray + module load PrgEnv-gnu/8.3.2 + module unload gcc + module load gcc/10.3.0 + module unload cray-mpich + module load cray-mpich/8.1.14 + module unload cray-python + module load cray-python/3.9.7.1 + module unload cray-libsci + module load cray-libsci/22.08.1.1 + + module use /p/app/projects/NEPTUNE/spack-stack/modulefiles + module load ecflow/5.8.4 + +For ``spack-stack-1.2.0``/``skylab-3.0.0`` with GNU, load the following modules after loading the above modules. + +**TEMPORARY** + +.. code-block:: console + + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-test-20230214/envs/skylab-dev-gnu-10.3.0/install/modulefiles/Core + module load stack-gcc/10.3.0 + module load stack-cray-mpich/8.1.14 + module load stack-python/3.9.7 + .. _Platforms_Casper: ------------------- From bb237276cd2b0d74531b11864f1099788f05316d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 15 Feb 2023 19:37:10 -0700 Subject: [PATCH 04/28] Add .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 96 ++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 .github/workflows/macos-ci-x86_64.yaml diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml new file mode 100644 index 000000000..982a7d3df --- /dev/null +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -0,0 +1,96 @@ +name: macos-ci-x86_64-build +on: + pull_request: + paths-ignore: + - 'doc/**' + - '**.md' + - '.github/ISSUE_TEMPLATE/*' + - '.gitignore' + workflow_dispatch: + # inputs: + # template: + # description: 'Base spack.yaml template. Default is a complete JEDI-UFS environment.' + # required: true + # default: 'skylab-dev' + # specs: + # description: 'Which specs to add to the template. Default is none (empty string).' + # required: false + # default: '' + + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +# Use custom shell with -l so .bash_profile is sourced +defaults: + run: + shell: bash -leo pipefail {0} + +jobs: + macos-dom-build: + #if: ${{ github.event.label.name == 'run-macos-dom' }} || ${{ inputs.template }} + runs-on: [self-hosted, macOS, X64, CI] + + steps: + - name: checkout + uses: actions/checkout@v2 + with: + submodules: true + + - name: create-env + run: | + source ./setup.sh + export ENVDIR=$PWD/${{ inputs.template || 'skylab-dev' }}.macos-ci-x86_64 + spack stack create env --site macos.default --template ${{ inputs.template || 'skylab-dev' }} --name ${ENVDIR} + spack env activate -d ${ENVDIR} + spack add ${{ inputs.specs || '' }} + export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" + + # Find external packages + spack external find --scope system + spack external find --scope system perl + spack external find --scope system wget + PATH="/usr/local/opt/curl/bin:$PATH" \ + spack external find --scope system curl + PATH="/usr/local/opt/qt5/bin:$PATH" \ + spack external find --scope system qt + spack external find --scope system texlive + # NOT YET + # spack external find --scope system mysql + + # Find compilers + spack compiler find --scope system + + export -n SPACK_SYSTEM_CONFIG_PATH + + spack config add "packages:all:providers:mpi:[openmpi@4.1.4]" + spack config add "packages:all:compiler:[apple-clang@14.0.0]" + sed -i '' "s/\['\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml + + spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0 + spack mirror add local file:///Users/ec2-user/spack-stack/source-cache/ + spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/ + + spack install --fail-fast --source --verbose 2>&1 | tee log.install.apple-clang-14.0.0 + # Missing: binary cache stuff + spack clean -a + spack module lmod refresh -y + spack stack setup-meta-modules + spack env deactivate + + - name: test-env + run: | + export ENVDIR=$PWD/${{ inputs.template || 'skylab-dev' }}.macos-ci-x86_64 + module use $ENVDIR/install/modulefiles/Core + module load stack-apple-clang/14.0.0 + module load stack-apple-clang/4.1.4 + module load stack-python/3.10.8 + module load jedi-ufs-env/1.0.0 jedi-ewok-env/1.0.0 soca-env/1.0.0 + + # Skip this for now, copied from other yaml + #- name: upload-mirror + # uses: actions/upload-artifact@v2 + # with: + # name: spack_mirror + # path: cache/source_cache From 706ea775c623a006817028374014c69fa0bb808a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 15 Feb 2023 19:42:42 -0700 Subject: [PATCH 05/28] Comment out wrong Intel libsci --- configs/sites/narwhal/compilers.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/sites/narwhal/compilers.yaml b/configs/sites/narwhal/compilers.yaml index f78e2c987..d00016a5b 100644 --- a/configs/sites/narwhal/compilers.yaml +++ b/configs/sites/narwhal/compilers.yaml @@ -34,8 +34,8 @@ compilers:: - PrgEnv-gnu/8.3.2 - gcc/10.3.0 environment: - prepend_path: - LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.08.1.1/INTEL/19.0/x86_64/lib' + #prepend_path: + # LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.08.1.1/INTEL/19.0/x86_64/lib' - NOT INTEL!!! set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] From a06eafe1aa17bdee575f9c562a0283a59a6b2dbd Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 15 Feb 2023 19:56:31 -0700 Subject: [PATCH 06/28] Update macOS CI scripts --- .github/workflows/macos-ci-x86_64.yaml | 2 +- .github/workflows/macos-dom.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index 982a7d3df..b7dea8010 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -30,7 +30,7 @@ defaults: jobs: macos-dom-build: #if: ${{ github.event.label.name == 'run-macos-dom' }} || ${{ inputs.template }} - runs-on: [self-hosted, macOS, X64, CI] + runs-on: [macos-ci-x86_64] steps: - name: checkout diff --git a/.github/workflows/macos-dom.yaml b/.github/workflows/macos-dom.yaml index b6bcb52c4..be3569dc5 100644 --- a/.github/workflows/macos-dom.yaml +++ b/.github/workflows/macos-dom.yaml @@ -27,7 +27,7 @@ defaults: jobs: macos-dom-build: if: ${{ github.event.label.name == 'run-macos-dom' }} || ${{ inputs.template }} - runs-on: [self-hosted, macOS, X64, macos-dom] + runs-on: [macos-dom] steps: - name: checkout From 4cfb4fc7b36efa06abe9642414fa32568413b87c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 16 Feb 2023 06:57:51 -0700 Subject: [PATCH 07/28] Bug fix in .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index b7dea8010..fd2798065 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -28,7 +28,7 @@ defaults: shell: bash -leo pipefail {0} jobs: - macos-dom-build: + macos-ci-x86_64-build: #if: ${{ github.event.label.name == 'run-macos-dom' }} || ${{ inputs.template }} runs-on: [macos-ci-x86_64] @@ -72,7 +72,7 @@ jobs: spack mirror add local file:///Users/ec2-user/spack-stack/source-cache/ spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/ - spack install --fail-fast --source --verbose 2>&1 | tee log.install.apple-clang-14.0.0 + spack install --fail-fast --source 2>&1 | tee log.install.apple-clang-14.0.0 # Missing: binary cache stuff spack clean -a spack module lmod refresh -y @@ -84,7 +84,7 @@ jobs: export ENVDIR=$PWD/${{ inputs.template || 'skylab-dev' }}.macos-ci-x86_64 module use $ENVDIR/install/modulefiles/Core module load stack-apple-clang/14.0.0 - module load stack-apple-clang/4.1.4 + module load stack-openmpi/4.1.4 module load stack-python/3.10.8 module load jedi-ufs-env/1.0.0 jedi-ewok-env/1.0.0 soca-env/1.0.0 From e98ede7bf7746f5091249c47e03ba7e383bb807e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 16 Feb 2023 13:40:08 -0700 Subject: [PATCH 08/28] Debug CI --- .github/workflows/macos-ci-x86_64.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index fd2798065..a921bf030 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -64,6 +64,10 @@ jobs: export -n SPACK_SYSTEM_CONFIG_PATH + # For buildcaches + spack config add config:padded_length:true + + # Set compiler and MPI spack config add "packages:all:providers:mpi:[openmpi@4.1.4]" spack config add "packages:all:compiler:[apple-clang@14.0.0]" sed -i '' "s/\['\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml @@ -72,6 +76,12 @@ jobs: spack mirror add local file:///Users/ec2-user/spack-stack/source-cache/ spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/ + # Break installation up in pieces and create buildcache in between + spack install --fail-fast --source base-env 2>&1 | tee log.install.base-env.apple-clang-14.0.0 + spack buildcache create -a -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index + + exit 1 + spack install --fail-fast --source 2>&1 | tee log.install.apple-clang-14.0.0 # Missing: binary cache stuff spack clean -a From c25f12db02f8249bc8c5c2cfc84d9dc829b3e2bb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 16 Feb 2023 14:30:42 -0700 Subject: [PATCH 09/28] Debug CI --- .github/workflows/macos-ci-x86_64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index a921bf030..5b80ff266 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -78,7 +78,7 @@ jobs: # Break installation up in pieces and create buildcache in between spack install --fail-fast --source base-env 2>&1 | tee log.install.base-env.apple-clang-14.0.0 - spack buildcache create -a -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index + spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index exit 1 From 33a1c73dd197255585417104d8601beddc4a3e66 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 16 Feb 2023 14:56:04 -0700 Subject: [PATCH 10/28] Update .gitmodules and submodule pointer for spack --- .gitmodules | 10 ++++++---- spack | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index c92fbbc5e..5d33a7156 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,11 @@ [submodule "spack"] path = spack - #url = https://github.com/spack/spack - #branch = develop - url = https://github.com/NOAA-EMC/spack - branch = jcsda_emc_spack_stack + ##url = https://github.com/spack/spack + ##branch = develop + #url = https://github.com/NOAA-EMC/spack + #branch = jcsda_emc_spack_stack + url = https://github.com/climbfuji/spack + branch = feature/mapl_basedir_buildcache_macos [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 83743d32a..eabe9b6f8 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 83743d32adb1ea1166248a4f64b93f5d4694c502 +Subproject commit eabe9b6f823dee90e92b6793d7d5d2294c63f1dc From 9dff00d0fa1e0f0985b3a357127a6ab619bc7ca1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 16 Feb 2023 15:27:04 -0700 Subject: [PATCH 11/28] Update .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index 5b80ff266..26f627783 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -73,12 +73,20 @@ jobs: sed -i '' "s/\['\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0 - spack mirror add local file:///Users/ec2-user/spack-stack/source-cache/ + + # Add and update source cache + spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/ spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/ - # Break installation up in pieces and create buildcache in between + # Add binary cache and reindex it + spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ + spack buildcache list + + exit 1 + + # Break installation up in pieces and create build caches in between spack install --fail-fast --source base-env 2>&1 | tee log.install.base-env.apple-clang-14.0.0 - spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index + spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/ --rebuild-index exit 1 From 613f21e4490732fd2defab7662f5d71d4f02fb59 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 16 Feb 2023 15:32:52 -0700 Subject: [PATCH 12/28] Update .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index 26f627783..3eb3d5ed2 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -80,13 +80,15 @@ jobs: # Add binary cache and reindex it spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ + spack buildcache update-index /Users/ec2-user/spack-stack/build-cache/ + echo "spack buildcache list:" spack buildcache list - + echo "exit" exit 1 # Break installation up in pieces and create build caches in between spack install --fail-fast --source base-env 2>&1 | tee log.install.base-env.apple-clang-14.0.0 - spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/ --rebuild-index + spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index exit 1 From 5e796752adccbc3d1d27f581fbf338b20869ed87 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 16 Feb 2023 15:36:27 -0700 Subject: [PATCH 13/28] Update .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index 3eb3d5ed2..7dd78a59e 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -80,7 +80,7 @@ jobs: # Add binary cache and reindex it spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ - spack buildcache update-index /Users/ec2-user/spack-stack/build-cache/ + spack buildcache update-index -d /Users/ec2-user/spack-stack/build-cache/ echo "spack buildcache list:" spack buildcache list echo "exit" From 39031e48ddc1a388c7d98296eeec83105940c0d2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 16 Feb 2023 16:05:43 -0700 Subject: [PATCH 14/28] Update .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index 7dd78a59e..b15bdf72f 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -80,7 +80,7 @@ jobs: # Add binary cache and reindex it spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ - spack buildcache update-index -d /Users/ec2-user/spack-stack/build-cache/ + spack buildcache update-index -m local-binary echo "spack buildcache list:" spack buildcache list echo "exit" From ebd11993981e639d248515a72b542fbdf76561dd Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 16 Feb 2023 20:18:15 -0700 Subject: [PATCH 15/28] Update .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index b15bdf72f..350eecf3f 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -81,19 +81,25 @@ jobs: # Add binary cache and reindex it spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ spack buildcache update-index -m local-binary - echo "spack buildcache list:" + echo "Packages in combined spack build caches:" spack buildcache list - echo "exit" - exit 1 # Break installation up in pieces and create build caches in between + # This allows us to "spin up" builds that altogether take longer than + # six hours, and/or fail later in the build process. + + # base-env spack install --fail-fast --source base-env 2>&1 | tee log.install.base-env.apple-clang-14.0.0 spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index + # jedi-base-env + spack install --fail-fast --source jedi-base-env 2>&1 | tee log.install.jedi-base-env.apple-clang-14.0.0 + spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index + # the rest + spack install --fail-fast --source 2>&1 | tee log.install.apple-clang-14.0.0 + spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index - exit 1 + # Next steps: synchronize source and build cache to a central/combined mirror? - spack install --fail-fast --source 2>&1 | tee log.install.apple-clang-14.0.0 - # Missing: binary cache stuff spack clean -a spack module lmod refresh -y spack stack setup-meta-modules From e1ef797897fa55377d66b20e49b70ea84103b17b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 16 Feb 2023 20:48:51 -0700 Subject: [PATCH 16/28] Trigger CI From be568ca69f2b0833e246053c8021138314ddc3b1 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 17 Feb 2023 05:59:42 -0700 Subject: [PATCH 17/28] Add --no-check-signature to spack install command in .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index 350eecf3f..ecc69e38f 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -89,13 +89,13 @@ jobs: # six hours, and/or fail later in the build process. # base-env - spack install --fail-fast --source base-env 2>&1 | tee log.install.base-env.apple-clang-14.0.0 + spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.base-env.apple-clang-14.0.0 spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index # jedi-base-env - spack install --fail-fast --source jedi-base-env 2>&1 | tee log.install.jedi-base-env.apple-clang-14.0.0 + spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.jedi-base-env.apple-clang-14.0.0 spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index # the rest - spack install --fail-fast --source 2>&1 | tee log.install.apple-clang-14.0.0 + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.0 spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index # Next steps: synchronize source and build cache to a central/combined mirror? From 9c2a2b1376259a670f247681fd7ac1919fc7d5dd Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 24 Feb 2023 19:13:18 -0700 Subject: [PATCH 18/28] Revert .gitmdoules and update submodule pointer for spack --- .gitmodules | 10 ++++------ spack | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5d33a7156..c92fbbc5e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,9 @@ [submodule "spack"] path = spack - ##url = https://github.com/spack/spack - ##branch = develop - #url = https://github.com/NOAA-EMC/spack - #branch = jcsda_emc_spack_stack - url = https://github.com/climbfuji/spack - branch = feature/mapl_basedir_buildcache_macos + #url = https://github.com/spack/spack + #branch = develop + url = https://github.com/NOAA-EMC/spack + branch = jcsda_emc_spack_stack [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index ba727e8aa..6df7bb3b1 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit ba727e8aa2738b5182e02d4c45180d5c448904c4 +Subproject commit 6df7bb3b1c45f29a1d191a52af24030bde64af8a From cb24419a52daf5850fba00c71069b15f5003a544 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 27 Feb 2023 11:56:06 -0700 Subject: [PATCH 19/28] Ignore non-zero return codes from spack buildcache create in .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index ecc69e38f..060dc0eb1 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -90,13 +90,20 @@ jobs: # base-env spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.base-env.apple-clang-14.0.0 + # For some reason the buildcache create step returns a non-zero status ... + set +eo pipefail spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index + set -eo pipefail # jedi-base-env spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.jedi-base-env.apple-clang-14.0.0 + set +eo pipefail spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index + set -eo pipefail # the rest spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.0 + set +eo pipefail spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index + set -eo pipefail # Next steps: synchronize source and build cache to a central/combined mirror? From 908e25872f123e5115da158aebab9b52a21bd733 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 27 Feb 2023 15:15:27 -0700 Subject: [PATCH 20/28] Debug .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index 060dc0eb1..81fe9fe1b 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -88,6 +88,8 @@ jobs: # This allows us to "spin up" builds that altogether take longer than # six hours, and/or fail later in the build process. + set -x + # base-env spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.base-env.apple-clang-14.0.0 # For some reason the buildcache create step returns a non-zero status ... From edd8a623198fa6aae48fc4f46d7ed3342315aa60 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 27 Feb 2023 15:25:31 -0700 Subject: [PATCH 21/28] Debug .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index 81fe9fe1b..0e5696e4f 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -91,18 +91,20 @@ jobs: set -x # base-env - spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.base-env.apple-clang-14.0.0 + spack install --fail-fast --source --no-check-signature base-env # For some reason the buildcache create step returns a non-zero status ... set +eo pipefail spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index set -eo pipefail + # jedi-base-env - spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.jedi-base-env.apple-clang-14.0.0 + spack install --fail-fast --source --no-check-signature jedi-base-env set +eo pipefail spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index set -eo pipefail + # the rest - spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.0 + spack install --fail-fast --source --no-check-signature set +eo pipefail spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index set -eo pipefail From 4a924e3aeff319881cbaf349a4c1a94d9a9972f7 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 27 Feb 2023 15:52:55 -0700 Subject: [PATCH 22/28] Debug .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index 0e5696e4f..f49947406 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -88,8 +88,6 @@ jobs: # This allows us to "spin up" builds that altogether take longer than # six hours, and/or fail later in the build process. - set -x - # base-env spack install --fail-fast --source --no-check-signature base-env # For some reason the buildcache create step returns a non-zero status ... From 4318c2f177bbc8e697fb50ffdffc005507497b63 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 27 Feb 2023 19:38:16 -0700 Subject: [PATCH 23/28] Use correct path to env dir in .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index f49947406..1a81addd7 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -41,7 +41,7 @@ jobs: - name: create-env run: | source ./setup.sh - export ENVDIR=$PWD/${{ inputs.template || 'skylab-dev' }}.macos-ci-x86_64 + export ENVDIR=$PWD/envs/${{ inputs.template || 'skylab-dev' }}.macos-ci-x86_64 spack stack create env --site macos.default --template ${{ inputs.template || 'skylab-dev' }} --name ${ENVDIR} spack env activate -d ${ENVDIR} spack add ${{ inputs.specs || '' }} @@ -116,7 +116,7 @@ jobs: - name: test-env run: | - export ENVDIR=$PWD/${{ inputs.template || 'skylab-dev' }}.macos-ci-x86_64 + export ENVDIR=$PWD/envs/${{ inputs.template || 'skylab-dev' }}.macos-ci-x86_64 module use $ENVDIR/install/modulefiles/Core module load stack-apple-clang/14.0.0 module load stack-openmpi/4.1.4 From 1e068cdd36f0c695c4a945c55fdb681699d3a142 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 27 Feb 2023 19:44:33 -0700 Subject: [PATCH 24/28] Debug CI --- .github/workflows/macos-ci-x86_64.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index 1a81addd7..fea2bd32a 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -33,6 +33,12 @@ jobs: runs-on: [macos-ci-x86_64] steps: + - name: cleanup + run: | + pwd + ls -lart + exit 1 + - name: checkout uses: actions/checkout@v2 with: From d83e6fb110e75b8d913a9af50f0d3edca06149d0 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 27 Feb 2023 19:46:37 -0700 Subject: [PATCH 25/28] Debug CI --- .github/workflows/macos-ci-x86_64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index fea2bd32a..25c088596 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -37,7 +37,7 @@ jobs: run: | pwd ls -lart - exit 1 + rm -fr * - name: checkout uses: actions/checkout@v2 From 095a834a35f42e67a544a41d3da7cb792dc80de4 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 28 Feb 2023 05:43:00 -0700 Subject: [PATCH 26/28] Source lmod to be able to run 'module use' etc --- .github/workflows/macos-ci-x86_64.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index 25c088596..cbc0d6940 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -122,6 +122,7 @@ jobs: - name: test-env run: | + source /usr/local/opt/lmod/init/profile export ENVDIR=$PWD/envs/${{ inputs.template || 'skylab-dev' }}.macos-ci-x86_64 module use $ENVDIR/install/modulefiles/Core module load stack-apple-clang/14.0.0 From 96b22f0e04c90252a8a536f2018720b78fbb1e0f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 28 Feb 2023 14:25:07 -0700 Subject: [PATCH 27/28] Add missing '--skip-on-error' to buildcache create commands --- .github/workflows/macos-ci-x86_64.yaml | 27 ++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index cbc0d6940..a8ec695b7 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -95,23 +95,26 @@ jobs: # six hours, and/or fail later in the build process. # base-env - spack install --fail-fast --source --no-check-signature base-env + spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.0.base-env # For some reason the buildcache create step returns a non-zero status ... - set +eo pipefail - spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index - set -eo pipefail + #set +eo pipefail + spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error + echo "Return code: '$?'" + #set -eo pipefail # jedi-base-env - spack install --fail-fast --source --no-check-signature jedi-base-env - set +eo pipefail - spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index - set -eo pipefail + spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.0.jedi-base-env + #set +eo pipefail + spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error + echo "Return code: '$?'" + #set -eo pipefail # the rest - spack install --fail-fast --source --no-check-signature - set +eo pipefail - spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --rebuild-index - set -eo pipefail + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.0.${{ inputs.template || 'skylab-dev' }} + #set +eo pipefail + spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error + echo "Return code: '$?'" + #set -eo pipefail # Next steps: synchronize source and build cache to a central/combined mirror? From 1ecdbcaf8c65e48bde36e47717abd715aeb3912c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 1 Mar 2023 07:41:20 -0700 Subject: [PATCH 28/28] Clean up .github/workflows/macos-ci-x86_64.yaml --- .github/workflows/macos-ci-x86_64.yaml | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/macos-ci-x86_64.yaml b/.github/workflows/macos-ci-x86_64.yaml index a8ec695b7..338dd4e6f 100644 --- a/.github/workflows/macos-ci-x86_64.yaml +++ b/.github/workflows/macos-ci-x86_64.yaml @@ -95,29 +95,31 @@ jobs: # six hours, and/or fail later in the build process. # base-env + echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.0.base-env - # For some reason the buildcache create step returns a non-zero status ... #set +eo pipefail - spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error - echo "Return code: '$?'" + spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error || true + #echo "Return code: '$?'" #set -eo pipefail - # jedi-base-env - spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.0.jedi-base-env - #set +eo pipefail - spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error - echo "Return code: '$?'" - #set -eo pipefail + ## jedi-base-env + #echo "jedi-base-env ..." + #spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.0.jedi-base-env + ##set +eo pipefail + #spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error || true + ##echo "Return code: '$?'" + ##set -eo pipefail # the rest + echo "${{ inputs.template || 'skylab-dev' }} ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.0.${{ inputs.template || 'skylab-dev' }} #set +eo pipefail - spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error - echo "Return code: '$?'" + spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error || true + #echo "Return code: '$?'" #set -eo pipefail # Next steps: synchronize source and build cache to a central/combined mirror? - + echo "Next steps ..." spack clean -a spack module lmod refresh -y spack stack setup-meta-modules