[develop] Integrate jinja-enabled config files and add rrfs workflow

.cicd/Jenkinsfile

@@ -13,7 +13,9 @@ pipeline {
         // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
         // Use the line below to enable the PW AWS cluster
         // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
-        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
+        // Use the line below to enable hera
+        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
+        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'jet', 'orion'], description: 'Specify the platform(s) to use')
         // Allow job runner to filter based on compiler
         choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
         // Uncomment the following line to re-enable comprehensive tests
@@ -76,7 +78,9 @@ pipeline {
                 axes {
                     axis {
                         name 'SRW_PLATFORM'
-                        values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                        // Uncomment line below to re-add use of Hera
+                        // values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                        values 'cheyenne', 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                     }
 
                     axis {
@@ -132,7 +136,7 @@ pipeline {
                         post {
                             success {
                                 sh 'cd "${WORKSPACE}/${INSTALL_NAME}" && tar --create --gzip --verbose --file "${WORKSPACE}/${BUILD_NAME}.tgz" *'
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build_${env.SRW_COMPILER}/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.log", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "${env.BUILD_NAME}.tgz", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: true, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: "build_${env.SRW_COMPILER}/srw_build-${env.SRW_PLATFORM}-${env.SRW_COMPILER}.txt", storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }
                     }
@@ -173,10 +177,10 @@ pipeline {
                         post {
                             always {
                                 // Archive the test log files
-                                sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --dereference --file "${WORKSPACE}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log.launch_FV3LAM_wflow */log/*'
+                                sh 'cd "${SRW_WE2E_EXPERIMENT_BASE_DIR}" && tar --create --gzip --verbose --dereference --file "${WORKSPACE}/we2e_test_logs-${SRW_PLATFORM}-${SRW_COMPILER}.tgz" */log.generate_FV3LAM_wflow */log/* ${WORKSPACE}/tests/WE2E/WE2E_tests_*yaml ${WORKSPACE}/tests/WE2E/WE2E_summary*txt ${WORKSPACE}/tests/WE2E/log.*'
                                 // Remove the data sets from the experiments directory to conserve disk space
                                 sh 'find "${SRW_WE2E_EXPERIMENT_BASE_DIR}" -regextype posix-extended -regex "^.*(orog|[0-9]{10})$" -type d | xargs rm -rf'
-                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'woc-epic-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_logs-*-*.tgz', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
+                                s3Upload consoleLogLevel: 'INFO', dontSetBuildResultOnFailure: false, dontWaitForConcurrentBuildCompletion: false, entries: [[bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_results-*-*.txt', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false], [bucket: 'noaa-epic-prod-jenkins-artifacts', excludedFile: '', flatten: false, gzipFiles: false, keepForever: false, managedArtifacts: true, noUploadOnFailure: false, selectedRegion: 'us-east-1', showDirectlyInBrowser: false, sourceFile: 'we2e_test_logs-*-*.tgz', storageClass: 'STANDARD', uploadFromSlave: false, useServerSideEncryption: false]], pluginFailureResultConstraint: 'FAILURE', profileName: 'main', userMetadata: []
                             }
                         }
                     }

.cicd/scripts/srw_build.sh

@@ -35,6 +35,6 @@ cd -
 # Create combined log file for upload to s3
 build_dir="${workspace}/build_${SRW_COMPILER}"
 cat ${build_dir}/log.cmake ${build_dir}/log.make \
-    >${build_dir}/srw_build-${platform}-${SRW_COMPILER}.log
+    >${build_dir}/srw_build-${platform}-${SRW_COMPILER}.txt
 
 exit $build_exit

.cicd/scripts/srw_test.sh

@@ -38,46 +38,17 @@ else
 fi
 
 cd ${we2e_test_dir}
-./setup_WE2E_tests.sh ${platform} ${SRW_PROJECT} ${SRW_COMPILER} ${test_type} \
-    expt_basedir=${we2e_experiment_base_dir} \
-    opsroot=${nco_dir}
-
-# Allow the tests to start before checking for status.
-# TODO: Create a parameter that sets the initial start delay.
-sleep 300
-
 # Progress file
 progress_file="${workspace}/we2e_test_results-${platform}-${SRW_COMPILER}.txt"
-
-# Wait for all tests to complete.
-while true; do
-
-    # Check status of all experiments
-    ./get_expts_status.sh expts_basedir="${we2e_experiment_base_dir}" \
-         verbose="FALSE" | tee ${progress_file}
-
-    # Exit loop only if there are not tests in progress
-    set +e
-    grep -q "Workflow status:  IN PROGRESS" ${progress_file}
-    exit_code=$?
-    set -e
-
-    if [[ $exit_code -ne 0 ]]; then
-       break
-    fi
-
-    # TODO: Create a paremeter that sets the poll frequency.
-    sleep 60
-done
-
-# Allow we2e cron jobs time to complete and clean up themselves
-# TODO: Create parameter that sets the interval for the we2e cron jobs; this
-# value should be some factor of that interval to ensure the cron jobs execute
-# before the workspace is cleaned up.
-sleep 600
+./setup_WE2E_tests.sh ${platform} ${SRW_PROJECT} ${SRW_COMPILER} ${test_type} \
+    --expt_basedir=${we2e_experiment_base_dir} \
+    --opsroot=${nco_dir} | tee ${progress_file}
 
 # Set exit code to number of failures
 set +e
-failures=$(grep "Workflow status:  FAILURE" ${progress_file} | wc -l)
+failures=$(grep " DEAD    " ${progress_file} | wc -l)
+if [[ $failures -ne 0 ]]; then
+    failures=1
+fi
 set -e
 exit ${failures}

.gitignore

@@ -5,13 +5,16 @@ fix/
 include/
 lib/
 share/
+modulefiles/extrn_comp_build/
 sorc/*/
-tests/WE2E/WE2E_test_info.csv
+tests/WE2E/WE2E_tests_*.yaml
 tests/WE2E/*.txt
 tests/WE2E/*.log
+tests/WE2E/log.*
 ush/__pycache__/
 ush/config.yaml
 ush/python_utils/__pycache__/
 ush/*.swp
 
 *.swp
+__pycache__

Externals.cfg

@@ -12,7 +12,7 @@ protocol = git
 repo_url = https://github.com/ufs-community/ufs-weather-model
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = 52072c5
+hash = 36d6e16
 local_path = sorc/ufs-weather-model
 required = True
 
@@ -21,7 +21,7 @@ protocol = git
 repo_url = https://github.com/NOAA-EMC/UPP
 # Specify either a branch name or a hash but not both.
 #branch = develop
-hash = a644aaa
+hash = 22cfb88
 local_path = sorc/UPP
 required = True
 
@@ -44,5 +44,23 @@ hash = 8d21f25
 local_path = sorc/rrfs_utl
 required = True
 
+[arl_nexus]
+protocol = git
+repo_url = https://github.com/noaa-oar-arl/NEXUS
+# Specify either a branch name or a hash but not both.
+#branch = develop
+hash = 3842818
+local_path = sorc/arl_nexus
+required = True
+
+[aqm-utils]
+protocol = git
+repo_url = https://github.com/NOAA-EMC/AQM-utils
+# Specify either a branch name or a hash but not both.
+#branch = develop
+hash = 0a86f73
+local_path = sorc/AQM-utils
+required = True
+
 [externals_description]
 schema_version = 1.0.0

README.md

@@ -1,15 +1,18 @@
 # UFS Short-Range Weather Application
 
-The Unified Forecast System (UFS) is a community-based, coupled, comprehensive Earth modeling system. It is designed to be the source system for NOAA’s operational numerical weather prediction applications while enabling research, development, and contribution opportunities for the broader weather enterprise. For more information about the UFS, visit the UFS Portal at https://ufscommunity.org/.
+The Unified Forecast System (UFS) is a community-based, coupled, comprehensive Earth modeling system. It is designed to be the source system for NOAA’s operational numerical weather prediction applications while enabling research, development, and contribution opportunities for the broader Weather Enterprise. For more information about the UFS, visit the UFS Portal at https://ufscommunity.org/.
 
-The UFS includes multiple applications (see a complete list at https://ufscommunity.org/science/aboutapps/) that support different forecast durations and spatial domains. This documentation describes the development branch of the UFS Short-Range Weather (SRW) Application, which targets predictions of atmospheric behavior on a limited spatial domain and on time scales from minutes to several days. The development branch of the application is continually evolving as the system undergoes open development. The latest SRW App release (v2.0.0) represents a snapshot of this continuously evolving system. 
+The UFS includes multiple applications (see a complete list at https://ufscommunity.org/science/aboutapps/) that support different forecast durations and spatial domains. This documentation describes the development branch of the UFS Short-Range Weather (SRW) Application, which targets predictions of atmospheric behavior on a limited spatial domain and on time scales from minutes to several days. The development branch of the application is continually evolving as the system undergoes open development. The latest SRW App release (v2.1.0) represents a snapshot of this continuously evolving system. 
 
-The UFS SRW App User's Guide associated with the development branch is at: https://ufs-srweather-app.readthedocs.io/en/develop/, while the guide specific to the SRW App v2.0.0 release can be found at: https://ufs-srweather-app.readthedocs.io/en/release-public-v2/. The repository is at: https://github.com/ufs-community/ufs-srweather-app.
+The UFS SRW App User's Guide associated with the development branch is at: https://ufs-srweather-app.readthedocs.io/en/develop/, while the guide specific to the SRW App v2.1.0 release can be found at: https://ufs-srweather-app.readthedocs.io/en/release-public-v2.1.0/. The repository is at: https://github.com/ufs-community/ufs-srweather-app.
 
 For instructions on how to clone the repository, build the code, and run the workflow, see:
 https://github.com/ufs-community/ufs-srweather-app/wiki/Getting-Started
 
-UFS Development Team. (2022, June 23). Unified Forecast System (UFS) Short-Range Weather (SRW) Application (Version v2.0.0). Zenodo. https://doi.org/10.5281/zenodo.6505854
+For a debugging guide for users and developers in the field of Earth System Modeling, please see:
+https://epic.noaa.gov/wp-content/uploads/2022/12/Debugging-Guide.pdf
+
+UFS Development Team. (2022, Nov. 17). Unified Forecast System (UFS) Short-Range Weather (SRW) Application (Version v2.1.0). Zenodo. https://doi.org/10.5281/zenodo.7277602
 
 [![Python unittests](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_unittests.yaml/badge.svg)](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_unittests.yaml)
 [![Python functional tests](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_func_tests.yaml/badge.svg)](https://github.com/ufs-community/ufs-srweather-app/actions/workflows/python_func_tests.yaml)

devbuild.sh

@@ -15,8 +15,8 @@ OPTIONS
       compiler to use; default depends on platform
       (e.g. intel | gnu | cray | gccgfortran)
   -a, --app=APPLICATION
-      weather model application to build
-      (e.g. ATM | ATMW | S2S | S2SW)
+      weather model application to build; for example, ATMAQ for Online-CMAQ
+      (e.g. ATM | ATMAQ | ATMW | S2S | S2SW)
   --ccpp="CCPP_SUITE1,CCPP_SUITE2..."
       CCPP suites (CCPP_SUITES) to include in build; delimited with ','
   --enable-options="OPTION1,OPTION2,..."
@@ -56,8 +56,7 @@ TARGETS
    all = builds all apps
    Or any combinations of (ufs, ufs_utils, upp, gsi, rrfs_utils)
 
-NOTE: This script is for internal developer use only;
-See User's Guide for detailed build instructions
+NOTE: See User's Guide for detailed build instructions
 
 EOF_USAGE
 }
@@ -87,6 +86,8 @@ Settings:
   BUILD_UPP=${BUILD_UPP}
   BUILD_GSI=${BUILD_GSI}
   BUILD_RRFS_UTILS=${BUILD_RRFS_UTILS}
+  BUILD_NEXUS=${BUILD_NEXUS}
+  BUILD_AQM_UTILS=${BUILD_AQM_UTILS}
 
 EOF_SETTINGS
 }
@@ -123,6 +124,8 @@ BUILD_UFS_UTILS="off"
 BUILD_UPP="off"
 BUILD_GSI="off"
 BUILD_RRFS_UTILS="off"
+BUILD_NEXUS="off"
+BUILD_AQM_UTILS="off"
 
 # Make options
 CLEAN=false
@@ -182,36 +185,51 @@ while :; do
     upp) DEFAULT_BUILD=false; BUILD_UPP="on" ;;
     gsi) DEFAULT_BUILD=false; BUILD_GSI="on" ;;
     rrfs_utils) DEFAULT_BUILD=false; BUILD_RRFS_UTILS="on" ;;
+    nexus) DEFAULT_BUILD=false; BUILD_NEXUS="on" ;;
+    aqm_utils) DEFAULT_BUILD=false; BUILD_AQM_UTILS="on" ;;
     # unknown
     -?*|?*) usage_error "Unknown option $1" ;;
     *) break
   esac
   shift
 done
 
-# choose default apps to build
-if [ "${DEFAULT_BUILD}" = true ]; then
-  BUILD_UFS="on"
-  BUILD_UFS_UTILS="on"
-  BUILD_UPP="on"
-fi
-
 # Ensure uppercase / lowercase ============================================
 APPLICATION="${APPLICATION^^}"
 PLATFORM="${PLATFORM,,}"
 COMPILER="${COMPILER,,}"
+EXTERNALS="${EXTERNALS^^}"
 
 # check if PLATFORM is set
 if [ -z $PLATFORM ] ; then
   printf "\nERROR: Please set PLATFORM.\n\n"
   usage
   exit 0
 fi
-
 # set PLATFORM (MACHINE)
 MACHINE="${PLATFORM}"
 printf "PLATFORM(MACHINE)=${PLATFORM}\n" >&2
 
+# choose default apps to build
+if [ "${DEFAULT_BUILD}" = true ]; then
+  BUILD_UFS="on"
+  BUILD_UFS_UTILS="on"
+  BUILD_UPP="on"
+fi
+
+# Choose components to build for air quality modeling (Online-CMAQ)
+if [ "${APPLICATION}" = "ATMAQ" ]; then
+  if [ "${DEFAULT_BUILD}" = true ]; then
+    BUILD_NEXUS="on"
+    BUILD_AQM_UTILS="on"
+  fi
+  if [ "${PLATFORM}" = "wcoss2" ]; then
+    BUILD_POST_STAT="on"
+  else
+    BUILD_POST_STAT="off"
+  fi
+fi
+
 set -eu
 
 # automatically determine compiler
@@ -221,8 +239,8 @@ if [ -z "${COMPILER}" ] ; then
     orion) COMPILER=intel ;;
     wcoss2) COMPILER=intel ;;
     cheyenne) COMPILER=intel ;;
-    macos,singularity) COMPILER=gnu ;;
-    odin,noaacloud) COMPILER=intel ;;
+    macos|singularity) COMPILER=gnu ;;
+    odin|noaacloud) COMPILER=intel ;;
     *)
       COMPILER=intel
       printf "WARNING: Setting default COMPILER=intel for new platform ${PLATFORM}\n" >&2;
@@ -304,7 +322,9 @@ CMAKE_SETTINGS="\
  -DBUILD_UFS_UTILS=${BUILD_UFS_UTILS}\
  -DBUILD_UPP=${BUILD_UPP}\
  -DBUILD_GSI=${BUILD_GSI}\
- -DBUILD_RRFS_UTILS=${BUILD_RRFS_UTILS}"
+ -DBUILD_RRFS_UTILS=${BUILD_RRFS_UTILS}\
+ -DBUILD_NEXUS=${BUILD_NEXUS}\
+ -DBUILD_AQM_UTILS=${BUILD_AQM_UTILS}"
 
 if [ ! -z "${APPLICATION}" ]; then
   CMAKE_SETTINGS="${CMAKE_SETTINGS} -DAPP=${APPLICATION}"
@@ -318,6 +338,29 @@ fi
 if [ ! -z "${DISABLE_OPTIONS}" ]; then
   CMAKE_SETTINGS="${CMAKE_SETTINGS} -DDISABLE_OPTIONS=${DISABLE_OPTIONS}"
 fi
+if [ "${APPLICATION}" = "ATMAQ" ]; then
+  CMAKE_SETTINGS="${CMAKE_SETTINGS} -DCPL_AQM=ON -DBUILD_POST_STAT=${BUILD_POST_STAT}"
+
+  # Copy module files to designated directory
+  EXTRN_BUILD_MOD_DIR="${SRW_DIR}/modulefiles/extrn_comp_build"
+  mkdir -p ${EXTRN_BUILD_MOD_DIR}
+  if [ "${BUILD_UFS}" = "on" ]; then
+    cp "${SRW_DIR}/sorc/ufs-weather-model/modulefiles/ufs_${PLATFORM}.${COMPILER}.lua" "${EXTRN_BUILD_MOD_DIR}/mod_ufs-weather-model.lua"
+    cp "${SRW_DIR}/sorc/ufs-weather-model/modulefiles/ufs_common.lua" ${EXTRN_BUILD_MOD_DIR}
+  fi
+  if [ "${BUILD_UFS_UTILS}" = "on" ]; then
+    cp "${SRW_DIR}/sorc/UFS_UTILS/modulefiles/build.${PLATFORM}.${COMPILER}.lua" "${EXTRN_BUILD_MOD_DIR}/mod_ufs-utils.lua"
+  fi
+  if [ "${BUILD_UPP}" = "on" ]; then
+    cp "${SRW_DIR}/sorc/UPP/modulefiles/${PLATFORM}.lua" "${EXTRN_BUILD_MOD_DIR}/mod_upp.lua" 
+  fi
+  if [ "${BUILD_NEXUS}" = "on" ]; then
+    cp "${SRW_DIR}/sorc/AQM-utils/parm/nexus_modulefiles/${PLATFORM}.${COMPILER}.lua" "${EXTRN_BUILD_MOD_DIR}/mod_nexus.lua"
+  fi
+  if [ "${BUILD_AQM_UTILS}" = "on" ]; then
+    cp "${SRW_DIR}/sorc/AQM-utils/modulefiles/build_${PLATFORM}.${COMPILER}.lua" "${EXTRN_BUILD_MOD_DIR}/mod_aqm-utils.lua"
+  fi
+fi
 
 # make settings
 MAKE_SETTINGS="-j ${BUILD_JOBS}"
@@ -393,6 +436,16 @@ if [ $USE_SUB_MODULES = true ]; then
         printf "... Loading RRFS_UTILS modules ...\n"
         load_module ""
     fi
+    if [ $BUILD_NEXUS = "on" ]; then
+        printf "... Loading NEXUS modules ...\n"
+        module use ${SRW_DIR}/sorc/arl_nexus/modulefiles
+        load_module ""
+    fi
+    if [ $BUILD_AQM_UTILS = "on" ]; then
+        printf "... Loading AQM-utils modules ...\n"
+        module use ${SRW_DIR}/sorc/AQM-utils/modulefiles
+        load_module ""
+    fi
 else
     module use ${SRW_DIR}/modulefiles
     module load ${MODULE_FILE}

docs/UsersGuide/requirements.txt

@@ -1,2 +1,3 @@
 sphinxcontrib-bibtex
 sphinx_rtd_theme
+docutils==0.16