Feature/update to gfdl 20210224

.github/actions/testing-setup/action.yml

@@ -0,0 +1,77 @@
+name: 'Build-.testing-prerequisites'
+description: 'Build pre-requisites for .testing including FMS and a symmetric MOM6 executable'
+inputs:
+  build_symmetric:
+    description: 'If true, will build the symmetric MOM6 executable'
+    required: false
+    default: 'true'
+  install_python:
+    description: 'If true, will install the local python env needed for .testing'
+    required: false
+    default: 'true'
+runs:
+  using: 'composite'
+  steps:
+    - name: Git info
+      shell: bash
+      run: |
+        echo "::group::Git commit info"
+        echo "git log:"
+        git log | head -60
+        echo "::endgroup::"
+
+    - name: Env
+      shell: bash
+      run: |
+        echo "::group::Environment"
+        env
+        echo "::endgroup::"
+
+    - name: Install needed packages for compiling
+      shell: bash
+      run: |
+        echo "::group::Install linux packages"
+        sudo apt-get update
+        sudo apt-get install netcdf-bin libnetcdf-dev libnetcdff-dev mpich libmpich-dev
+        echo "::endgroup::"
+
+    - name: Compile FMS library
+      shell: bash
+      run: |
+        echo "::group::Compile FMS library"
+        cd .testing
+        make deps/lib/libFMS.a -s -j
+        echo "::endgroup::"
+
+    - name: Store compiler flags used in Makefile
+      shell: bash
+      run: |
+        echo "::group::config.mk"
+        cd .testing
+        echo "FCFLAGS_DEBUG=-g -O0 -Wextra -Wno-compare-reals -fbacktrace -ffpe-trap=invalid,zero,overflow -fcheck=bounds" >> config.mk
+        echo "FCFLAGS_REPRO=-g -O2 -fbacktrace" >> config.mk
+        echo "FCFLAGS_INIT=-finit-real=snan -finit-integer=2147483647 -finit-derived" >> config.mk
+        echo "FCFLAGS_COVERAGE=--coverage" >> config.mk
+        cat config.mk
+        echo "::endgroup::"
+
+    - name: Compile MOM6 in symmetric memory mode
+      shell: bash
+      run: |
+        echo "::group::Compile MOM6 in symmetric memory mode"
+        cd .testing
+        test ${{ inputs.build_symmetric }} == true && make build/symmetric/MOM6 -j
+        echo "::endgroup::"
+
+    - name: Install local python venv for generating input data
+      shell: bash
+      run: |
+        echo "::group::Create local python env for input data generation"
+        cd .testing
+        test ${{ inputs.install_python }} == true && make work/local-env
+        echo "::endgroup::"
+
+    - name: Set flags
+      shell: bash
+      run: |
+        echo "TIMEFORMAT=... completed in %lR  (user: %lU, sys: %lS)" >> $GITHUB_ENV

.github/workflows/coupled-api.yml

@@ -0,0 +1,33 @@
+name: API for coupled drivers
+
+on: [push, pull_request]
+
+jobs:
+  test-top-api:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+      with:
+        build_symmetric: 'false'
+        install_python: 'false'
+
+    - name: Compile MOM6 for the GFDL coupled driver
+      shell: bash
+      run: make check_mom6_api_coupled -j
+
+    - name: Compile MOM6 for the NUOPC driver
+      shell: bash
+      run: make check_mom6_api_nuopc -j
+
+    - name: Compile MOM6 for the MCT driver
+      shell: bash
+      run: make check_mom6_api_mct -j

.github/workflows/coverage.yml

@@ -0,0 +1,24 @@
+name: Code coverage
+
+on: [push, pull_request]
+
+jobs:
+  build-test-nans:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    env:
+      REPORT_COVERAGE: true
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+
+    - name: Run and post coverage
+      run: make run.symmetric -k -s

.github/workflows/documentation-and-style.yml

@@ -0,0 +1,39 @@
+name: Doxygen and style
+
+on: [push, pull_request]
+
+jobs:
+  doxygen:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - name: Check white space (non-blocking)
+      run: |
+        ./.testing/trailer.py -e TEOS10 -l 120 src config_src 2>&1 | tee style_errors
+      continue-on-error: true
+
+    - name: Install packages used when generating documentation
+      run: |
+        sudo apt-get update
+        sudo apt-get install python3-sphinx python3-lxml perl
+        sudo apt-get install texlive-binaries texlive-base bibtool tex-common texlive-bibtex-extra
+        sudo apt-get install graphviz
+
+    - name: Build doxygen HTML
+      run: |
+        cd docs
+        perl -e 'print "perl version $^V" . "\n"'
+        mkdir _build && make nortd DOXYGEN_RELEASE=Release_1_8_13 UPDATEHTMLEQS=Y
+        cat _build/doxygen_warn_nortd_log.txt
+
+    - name: Report doxygen or style errors
+      run: |
+        grep "warning:" docs/_build/doxygen_warn_nortd_log.txt | grep -v "as part of a" | tee doxy_errors
+        cat style_errors doxy_errors > all_errors
+        cat all_errors
+        test ! -s  all_errors

.github/workflows/expression.yml

@@ -0,0 +1,27 @@
+name: Expression verification
+
+on: [push, pull_request]
+
+jobs:
+  test-repro-and-dims:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+
+    - name: Compile MOM6 using repro optimization
+      run: make build/repro/MOM6 -j
+
+    - name: Create validation data
+      run: make run.symmetric -k -s
+
+    - name: Run tests
+      run: make test.repros test.dims -k -s

.github/workflows/other.yml

@@ -0,0 +1,27 @@
+name: OpenMP and Restart verification
+
+on: [push, pull_request]
+
+jobs:
+  test-openmp-nan-restarts:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+
+    - name: Compile with openMP
+      run: make build/openmp/MOM6 -j
+
+    - name: Create validation data
+      run: make run.symmetric -k -s
+
+    - name: Run tests
+      run: make test.openmps test.nans test.restarts -k -s

.github/workflows/regression.yml

@@ -0,0 +1,27 @@
+name: Regression
+
+on: [pull_request]
+
+jobs:
+  build-test-regression:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+
+    - name: Compile reference model
+      run: make build.regressions MOM_TARGET_SLUG=$GITHUB_REPOSITORY MOM_TARGET_LOCAL_BRANCH=$GITHUB_BASE_REF DO_REGRESSION_TESTS=true -j
+
+    - name: Create validation data
+      run: make run.symmetric -k -s
+
+    - name: Regression test
+      run: make test.regressions DO_REGRESSION_TESTS=true -k -s

.github/workflows/stencil.yml

@@ -0,0 +1,27 @@
+name: Stencil related verification
+
+on: [push, pull_request]
+
+jobs:
+  test-symmetric-layout-rotation:
+
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .testing
+
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: recursive
+
+    - uses: ./.github/actions/testing-setup
+
+    - name: Compile MOM6 in asymmetric memory mode
+      run: make build/asymmetric/MOM6 -j
+
+    - name: Create validation data
+      run: make run.symmetric -k -s
+
+    - name: Run tests
+      run: make test.grids test.layouts test.rotations -k -s

.gitignore

@@ -4,13 +4,7 @@
 html
 
 
-# Build output
-*.o
-*.mod
-MOM6
-
-
-# Autoconf
+# Autoconf output
 aclocal.m4
 autom4te.cache/
 config.log

.gitlab-ci.yml

@@ -117,11 +117,36 @@ run:
     - time tar zxf $CACHE_DIR/build-pgi-repro-$CI_PIPELINE_ID.tgz
     # time tar zxf $CACHE_DIR/build-gnu-debug-$CI_PIPELINE_ID.tgz
     - (echo '#!/bin/tcsh';echo 'make -f MRS/Makefile.tests all') > job.sh
-    - sbatch --clusters=c3,c4 --nodes=29 --time=0:34:00 --account=gfdl_o --qos=debug --job-name=mom6_regressions --output=log.$CI_PIPELINE_ID --wait job.sh
+    - sbatch --clusters=c3,c4 --nodes=29 --time=0:34:00 --account=gfdl_o --qos=debug --job-name=mom6_regressions --output=log.$CI_PIPELINE_ID --wait job.sh || MJOB_RETURN_STATE=Fail
     - cat log.$CI_PIPELINE_ID
+    - test -z "$MJOB_RETURN_STATE"
     - test -f restart_results_gnu.tar.gz
     - time tar zvcf $CACHE_DIR/results-$CI_PIPELINE_ID.tgz *.tar.gz
 
+gnu.testing:
+  stage: run
+  tags:
+    - ncrc4
+  script:
+    - cd .testing
+    - module unload PrgEnv-pgi PrgEnv-intel PrgEnv-gnu darshan ; module load PrgEnv-gnu ; module unload netcdf gcc ; module load gcc/7.3.0 cray-hdf5 cray-netcdf
+    - make work/local-env
+    - make -s -j
+    - (echo '#!/bin/bash';echo '. ./work/local-env/bin/activate';echo 'make MPIRUN="srun -mblock --exclusive" test -s -j') > job.sh
+    - sbatch --clusters=c3,c4 --nodes=5 --time=0:05:00 --account=gfdl_o --qos=debug --job-name=MOM6.gnu.testing --output=log.$CI_PIPELINE_ID --wait job.sh || cat log.$CI_PIPELINE_ID && make test
+
+intel.testing:
+  stage: run
+  tags:
+    - ncrc4
+  script:
+    - cd .testing
+    - module unload PrgEnv-pgi PrgEnv-intel PrgEnv-gnu darshan; module load PrgEnv-intel; module unload netcdf intel; module load intel/18.0.6.288 cray-hdf5 cray-netcdf
+    - make work/local-env
+    - make -s -j
+    - (echo '#!/bin/bash';echo '. ./work/local-env/bin/activate';echo 'make MPIRUN="srun -mblock --exclusive" test -s -j') > job.sh
+    - sbatch --clusters=c3,c4 --nodes=5 --time=0:05:00 --account=gfdl_o --qos=debug --job-name=MOM6.gnu.testing --output=log.$CI_PIPELINE_ID --wait job.sh || cat log.$CI_PIPELINE_ID && make test
+
 # Tests
 gnu:non-symmetric:
   stage: tests

.readthedocs.yml

@@ -1,11 +1,16 @@
-# don't build extra formats (like HTML zip)
-formats:
-  - none
+version: 2
 
-# path to pip requirements file to bring in
-# doxygen extensions
-requirements_file: docs/requirements.txt
+# Extra formats
+# PDF generation is failing for now; disabled on 2020-12-02
+#formats:
+#  - pdf
+
+# Build documentation
+sphinx:
+  configuration: docs/conf.py
 
 python:
   # make sure we're using Python 3
-  version: 3.5
+  version: 3
+  install:
+    - requirements: docs/requirements.txt