Add Narwhal GNU site config, add self-hosted CI runner on macOS

.github/workflows/macos-ci-x86_64.yaml

@@ -0,0 +1,143 @@
+name: macos-ci-x86_64-build
+on:
+  pull_request:
+    paths-ignore:
+      - 'doc/**'
+      - '**.md'
+      - '.github/ISSUE_TEMPLATE/*'
+      - '.gitignore'
+  workflow_dispatch:
+  #  inputs:
+  #    template:
+  #      description: 'Base spack.yaml template. Default is a complete JEDI-UFS environment.'
+  #      required: true
+  #      default: 'skylab-dev'
+  #    specs:
+  #      description: 'Which specs to add to the template. Default is none (empty string).'
+  #      required: false
+  #      default: ''
+
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+# Use custom shell with -l so .bash_profile is sourced
+defaults:
+  run:
+    shell: bash -leo pipefail {0}
+
+jobs:
+  macos-ci-x86_64-build:
+    #if: ${{ github.event.label.name == 'run-macos-dom' }} || ${{ inputs.template }}
+    runs-on: [macos-ci-x86_64]
+
+    steps:
+      - name: cleanup
+        run: |
+          pwd
+          ls -lart
+          rm -fr *
+
+      - name: checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: create-env
+        run: |
+          source ./setup.sh
+          export ENVDIR=$PWD/envs/${{ inputs.template || 'skylab-dev' }}.macos-ci-x86_64
+          spack stack create env --site macos.default --template ${{ inputs.template || 'skylab-dev' }} --name ${ENVDIR}
+          spack env activate -d ${ENVDIR}
+          spack add ${{ inputs.specs || '' }}
+          export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
+
+          # Find external packages
+          spack external find --scope system
+          spack external find --scope system perl
+          spack external find --scope system wget
+          PATH="/usr/local/opt/curl/bin:$PATH" \
+              spack external find --scope system curl
+          PATH="/usr/local/opt/qt5/bin:$PATH" \
+              spack external find --scope system qt
+          spack external find --scope system texlive
+          # NOT YET
+          # spack external find --scope system mysql
+
+          # Find compilers
+          spack compiler find --scope system
+
+          export -n SPACK_SYSTEM_CONFIG_PATH
+
+          # For buildcaches
+          spack config add config:padded_length:true
+
+          # Set compiler and MPI
+          spack config add "packages:all:providers:mpi:[openmpi@4.1.4]"
+          spack config add "packages:all:compiler:[apple-clang@14.0.0]"
+          sed -i '' "s/\['\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml
+
+          spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.0
+
+          # Add and update source cache
+          spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/
+          spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/
+
+          # Add binary cache and reindex it
+          spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/
+          spack buildcache update-index -m local-binary
+          echo "Packages in combined spack build caches:"
+          spack buildcache list
+
+          # Break installation up in pieces and create build caches in between
+          # This allows us to "spin up" builds that altogether take longer than
+          # six hours, and/or fail later in the build process.
+
+          # base-env
+          echo "base-env ..."
+          spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.0.base-env
+          #set +eo pipefail
+          spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error || true
+          #echo "Return code: '$?'"
+          #set -eo pipefail
+
+          ## jedi-base-env
+          #echo "jedi-base-env ..."
+          #spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.0.jedi-base-env
+          ##set +eo pipefail
+          #spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error || true
+          ##echo "Return code: '$?'"
+          ##set -eo pipefail
+
+          # the rest
+          echo "${{ inputs.template || 'skylab-dev' }} ..."
+          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.0.${{ inputs.template || 'skylab-dev' }}
+          #set +eo pipefail
+          spack buildcache create -a -r -u -d /Users/ec2-user/spack-stack/build-cache/ --skip-on-error || true
+          #echo "Return code: '$?'"
+          #set -eo pipefail
+
+          # Next steps: synchronize source and build cache to a central/combined mirror?
+          echo "Next steps ..."
+          spack clean -a
+          spack module lmod refresh -y
+          spack stack setup-meta-modules
+          spack env deactivate
+
+      - name: test-env
+        run: |
+          source /usr/local/opt/lmod/init/profile
+          export ENVDIR=$PWD/envs/${{ inputs.template || 'skylab-dev' }}.macos-ci-x86_64
+          module use $ENVDIR/install/modulefiles/Core
+          module load stack-apple-clang/14.0.0
+          module load stack-openmpi/4.1.4
+          module load stack-python/3.10.8
+          module load jedi-ufs-env/1.0.0 jedi-ewok-env/1.0.0 soca-env/1.0.0
+
+      # Skip this for now, copied from other yaml
+      #- name: upload-mirror
+      #  uses: actions/upload-artifact@v2
+      #  with:
+      #    name: spack_mirror
+      #    path: cache/source_cache

.github/workflows/macos-dom.yaml

@@ -27,7 +27,7 @@ defaults:
 jobs:
   macos-dom-build:
     if: ${{ github.event.label.name == 'run-macos-dom' }} || ${{ inputs.template }}
-    runs-on: [self-hosted, macOS, X64, macos-dom]
+    runs-on: [macos-dom]
 
     steps:
       - name: checkout

configs/sites/narwhal/compilers.yaml

@@ -20,3 +20,22 @@ compilers::
         set:
           CRAYPE_LINK_TYPE: 'dynamic'
       extra_rpaths: []
+  - compiler:
+      spec: gcc@10.3.0
+      paths:
+        cc: cc
+        cxx: CC
+        f77: ftn
+        fc: ftn
+      flags: {}
+      operating_system: sles15
+      target: any
+      modules:
+      - PrgEnv-gnu/8.3.2
+      - gcc/10.3.0
+      environment:
+        prepend_path:
+          LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.08.1.1/GNU/9.1/x86_64/lib'
+        set:
+          CRAYPE_LINK_TYPE: 'dynamic'
+      extra_rpaths: []

configs/sites/narwhal/packages.yaml

@@ -1,6 +1,6 @@
 packages:
   all:
-    compiler:: [intel@2021.3.0]
+    compiler:: [intel@2021.3.0, gcc@10.3.0]
     providers:
       mpi:: [cray-mpich@8.1.14]
 
@@ -12,6 +12,9 @@ packages:
     - spec: cray-mpich@8.1.14%intel@2021.3.0 ~wrappers
       modules:
       - cray-mpich/8.1.14
+    - spec: cray-mpich@8.1.14%gcc@10.3.0 ~wrappers
+      modules:
+      - cray-mpich/8.1.14
   #intel-oneapi-tbb:
   #  externals:
   #  - spec: intel-oneapi-tbb@2021.3.0

doc/source/Platforms.rst

@@ -31,7 +31,9 @@ spack-stack-v1
 +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+
 | NASA Discover GNU                                        | Dom Heinzeller            | ``/discover/swdev/jcsda/spack-stack/spack-stack-v1/envs/skylab-3.0.0-gnu-10.1.0/install``                          |
 +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+
-| NAVY HPCMP Narwhal                                       | Dom Heinzeller            | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.4.0/install``                    |
+| NAVY HPCMP Narwhal Intel                                 | Dom Heinzeller            | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-2021.4.0/install``                    |
++----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+
+| NAVY HPCMP Narwhal GNU (**TEMPORARY LOCATION**)          | Dom Heinzeller            | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-test-20230214/envs/skylab-dev-gnu-10.3.0-libsci-22.08.1.1/install``|
 +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+
 | NCAR-Wyoming Casper                                      | Dom Heinzeller            | ``/glade/work/jedipara/cheyenne/spack-stack/spack-stack-v1/envs/skylab-3.0.0-intel-19.1.1.217-casper/install``     |
 +----------------------------------------------------------+---------------------------+--------------------------------------------------------------------------------------------------------------------+
@@ -144,7 +146,7 @@ For ``spack-stack-1.2.0``/``skylab-3.0.0`` with GNU, load the following modules
 NAVY HPCMP Narwhal
 ------------------------------
 
-The following is required for building new spack environments and for using spack to build and run software.
+With Intel, the following is required for building new spack environments and for using spack to build and run software:
 
 .. code-block:: console
 
@@ -171,6 +173,33 @@ For ``spack-stack-1.2.0``/``skylab-3.0.0`` with Intel, load the following module
    module load stack-cray-mpich/8.1.14
    module load stack-python/3.9.7
 
+With GNU, the following is required for building new spack environments and for using spack to build and run software:
+
+.. code-block:: console
+
+   module unload PrgEnv-cray
+   module load PrgEnv-gnu/8.3.2
+   module unload gcc
+   module load gcc/10.3.0
+   module unload cray-mpich
+   module load cray-mpich/8.1.14
+   module unload cray-python
+   module load cray-python/3.9.7.1
+   module unload cray-libsci
+   module load cray-libsci/22.08.1.1
+
+   module use /p/app/projects/NEPTUNE/spack-stack/modulefiles
+   module load ecflow/5.8.4
+
+For ``spack-stack-1.2.0``/``skylab-3.0.0`` with GNU, load the following modules after loading the above modules. **Note: temporary location!**
+
+.. code-block:: console
+
+   module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-test-20230214/envs/skylab-dev-gnu-10.3.0-libsci-22.08.1.1/install/modulefiles/Core
+   module load stack-gcc/10.3.0
+   module load stack-cray-mpich/8.1.14
+   module load stack-python/3.9.7
+
 .. _Platforms_Casper:
 
 -------------------
@@ -462,11 +491,11 @@ It is also instructive to peruse the GitHub actions scripts in ``.github/workflo
 macOS
 ------------------------------
 
-On macOS, it is important to use certain Homebrew packages as external packages, because the native macOS packages are incomplete (e.g. missing the development header files): ``curl``, ``python``, ``qt``, etc. The instructions provided in the following have been tested extensively on many macOS installations.
+On macOS, it is important to use certain Homebrew packages as external packages, because the native macOS packages are incomplete (e.g. missing the development header files): ``curl``, ``qt``, etc. The instructions provided in the following have been tested extensively on many macOS installations.
 
-The instructions below also assume a clean Homebrew installation with a clean Python installation inside. This means that the Homebrew Python only contains nothing but what gets installed with ``pip install poetry`` (which is a temporary workaround). If this is not the case, users can try to install a separate Python using Miniconda as described in :numref:`Sections %s <Prerequisites_Miniconda>`.
+Unlike in previous versions, the instructions below assume that ``Python`` is built by ``spack``. That means that when using the ``spack`` environments (i.e., loading the modules for building or running code), the ``spack`` installation of ``Python`` with its available ``Python`` modules should be used to ensure consistency. However, a Homebrew ``Python`` installation may still be needed to build new ``spack`` environments. It can also be beneficial for the user to have a version of ``Python`` installed with Homebrew that can be used for virtual environments that are completely independent of any ``spack``-built environment.
 
-Further, it is recommended to not use ``mpich`` or ``openmpi`` installed by Homebrew, because these packages are built using a flat namespace that is incompatible with the JEDI software. The spack-stack installations of ``mpich`` and ``openmpi`` use two-level namespaces as required.
+It is recommended to not use ``mpich`` or ``openmpi`` installed by Homebrew, because these packages are built using a flat namespace that is incompatible with the JEDI software. The spack-stack installations of ``mpich`` and ``openmpi`` use two-level namespaces as required.
 
 Intel M1 platform notes
 -----------------------
@@ -654,7 +683,7 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear
 
    unset SPACK_SYSTEM_CONFIG_PATH
 
-6. Set default compiler and MPI library and flag Python as non-buildable (make sure to use the correct ``apple-clang`` version for your system and the desired ``openmpi`` version)
+6. Set default compiler and MPI library (make sure to use the correct ``apple-clang`` version for your system and the desired ``openmpi`` version)
 
 .. code-block:: console
 
@@ -746,9 +775,6 @@ The following instructions were used to prepare a basic Red Hat 8 system as it i
    # Python
    yum -y install python39-devel
    alternatives --set python3 /usr/bin/python3.9
-   python3 -m pip install poetry
-   # test - successful if no output
-   python3 -c "import poetry"
 
    # Exit root session
    exit
@@ -800,9 +826,6 @@ The following instructions were used to prepare a basic Ubuntu 20.04 system as i
 
    # Python
    apt install -y python3-dev python3-pip
-   python3 -m pip install poetry
-   # test - successful if no output
-   python3 -c "import poetry"
 
    # Exit root session
    exit
@@ -850,9 +873,6 @@ The following instructions were used to prepare a basic Ubuntu 22.04 system as i
 
    # Python
    apt install -y python3-dev python3-pip
-   python3 -m pip install poetry
-   # test - successful if no output
-   python3 -c "import poetry"
 
    # Exit root session
    exit
@@ -887,7 +907,8 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi
 
    spack external find --scope system
    spack external find --scope system perl
-   spack external find --scope system python
+   # Don't use any external Python, let spack build it
+   #spack external find --scope system python
    spack external find --scope system wget
    spack external find --scope system texlive
    # On Ubuntu (but not on Red Hat):
@@ -905,23 +926,20 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi
 
    unset SPACK_SYSTEM_CONFIG_PATH
 
-6. Set default compiler and MPI library and flag Python as non-buildable (make sure to use the correct ``gcc`` version for your system and the desired ``openmpi`` version)
+6. Set default compiler and MPI library (make sure to use the correct ``gcc`` version for your system and the desired ``openmpi`` version)
 
 .. code-block:: console
 
    # Example for Red Hat 8 following the above instructions
-   spack config add "packages:python:buildable:False"
    spack config add "packages:all:providers:mpi:[openmpi@4.1.4]"
    spack config add "packages:all:compiler:[gcc@11.2.1]"
 
    # Example for Ubuntu 20.04 following the above instructions
-   spack config add "packages:python:buildable:False"
    spack config add "packages:all:providers:mpi:[mpich@4.0.2]"
    spack config add "packages:all:compiler:[gcc@10.3.0]"
 
    # Example for Ubuntu 22.04 following the above instructions
    sed -i 's/tcl/lmod/g' envs/jedi-ufs.mylinux/site/modules.yaml
-   spack config add "packages:python:buildable:False"
    spack config add "packages:all:providers:mpi:[mpich@4.0.2]"
    spack config add "packages:all:compiler:[gcc@11.2.0]"