diff --git a/Externals.cfg b/Externals.cfg index ce380cfd32..7c9e1f1eb2 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -3,7 +3,7 @@ protocol = git repo_url = https://github.com/ufs-community/regional_workflow # Specify either a branch name or a hash but not both. #branch = develop -hash = de82b63 +branch = release/public-v2 local_path = regional_workflow required = True @@ -18,10 +18,11 @@ required = True [ufs-weather-model] protocol = git -repo_url = https://github.com/ufs-community/ufs-weather-model +#repo_url = https://github.com/ufs-community/ufs-weather-model +repo_url = https://github.com/NOAA-EPIC/ufs-weather-model-1 # Specify either a branch name or a hash but not both. #branch = develop -hash = 96dffa1 +branch = release-public-v3-SRW local_path = src/ufs-weather-model required = True diff --git a/devbuild.sh b/devbuild.sh index aaad8dd09d..bc81e4c8d5 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -10,15 +10,15 @@ OPTIONS show this help guide -p, --platform=PLATFORM name of machine you are building on - (e.g. cheyenne | hera | jet | orion | wcoss_dell_p3) + (e.g. cheyenne | hera | jet | orion | wcoss_dell_p3 | wcoss2) -c, --compiler=COMPILER compiler to use; default depends on platform (e.g. intel | gnu | cray | gccgfortran) - --app=APPLICATION + -a, --app=APPLICATION weather model application to build (e.g. ATM | ATMW | S2S | S2SW) --ccpp="CCPP_SUITE1,CCPP_SUITE2..." - CCCP suites to include in build; delimited with ',' + CCPP suites (CCPP_SUITES) to include in build; delimited with ',' --enable-options="OPTION1,OPTION2,..." enable ufs-weather-model options; delimited with ',' (e.g. 32BIT | INLINE_POST | UFS_GOCART | MOM6 | CICE6 | WW3 | CMEPS) @@ -52,13 +52,13 @@ settings () { cat << EOF_SETTINGS Settings: - SRC_DIR=${SRC_DIR} + SRW_DIR=${SRW_DIR} BUILD_DIR=${BUILD_DIR} INSTALL_DIR=${INSTALL_DIR} PLATFORM=${PLATFORM} COMPILER=${COMPILER} APP=${APPLICATION} - CCPP=${CCPP} + CCPP=${CCPP_SUITES} ENABLE_OPTIONS=${ENABLE_OPTIONS} DISABLE_OPTIONS=${DISABLE_OPTIONS} CLEAN=${CLEAN} @@ -79,13 +79,13 @@ usage_error () { # default settings LCL_PID=$$ -SRC_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) -MACHINE_SETUP=${SRC_DIR}/src/UFS_UTILS/sorc/machine-setup.sh -BUILD_DIR=${SRC_DIR}/build -INSTALL_DIR=${SRC_DIR} +SRW_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) +MACHINE_SETUP=${SRW_DIR}/src/UFS_UTILS/sorc/machine-setup.sh +BUILD_DIR="${SRW_DIR}/build" +INSTALL_DIR=${SRW_DIR} COMPILER="" APPLICATION="" -CCPP="" +CCPP_SUITES="" ENABLE_OPTIONS="" DISABLE_OPTIONS="" BUILD_TYPE="RELEASE" @@ -108,9 +108,9 @@ while :; do --platform|--platform=|-p|-p=) usage_error "$1 requires argument." ;; --compiler=?*|-c=?*) COMPILER=${1#*=} ;; --compiler|--compiler=|-c|-c=) usage_error "$1 requires argument." ;; - --app=?*) APPLICATION=${1#*=} ;; - --app|--app=) usage_error "$1 requires argument." ;; - --ccpp=?*) CCPP=${1#*=} ;; + --app=?*|-a=?*) APPLICATION=${1#*=} ;; + --app|--app=|-a|-a=) usage_error "$1 requires argument." ;; + --ccpp=?*) CCPP_SUITES=${1#*=} ;; --ccpp|--ccpp=) usage_error "$1 requires argument." ;; --enable-options=?*) ENABLE_OPTIONS=${1#*=} ;; --enable-options|--enable-options=) usage_error "$1 requires argument." ;; @@ -136,6 +136,11 @@ while :; do shift done +# Ensure uppercase / lowercase ============================================ +APPLICATION="${APPLICATION^^}" +PLATFORM="${PLATFORM,,}" +COMPILER="${COMPILER,,}" + # check if PLATFORM is set if [ -z $PLATFORM ] ; then printf "\nERROR: Please set PLATFORM.\n\n" @@ -155,9 +160,10 @@ if [ -z "${COMPILER}" ] ; then jet|hera|gaea) COMPILER=intel ;; orion|noaacloud) COMPILER=intel ;; wcoss_dell_p3) COMPILER=intel ;; + wcoss2) COMPILER=intel ;; cheyenne) COMPILER=intel ;; macos,singularity) COMPILER=gnu ;; - odin) COMPILER=intel ;; + odin,noaacloud) COMPILER=intel ;; *) COMPILER=intel printf "WARNING: Setting default COMPILER=intel for new platform ${PLATFORM}\n" >&2; @@ -174,7 +180,7 @@ fi # set MODULE_FILE for this platform/compiler combination MODULE_FILE="build_${PLATFORM}_${COMPILER}" -if [ ! -f "${SRC_DIR}/modulefiles/${MODULE_FILE}" ]; then +if [ ! -f "${SRW_DIR}/modulefiles/${MODULE_FILE}" ]; then printf "ERROR: module file does not exist for platform/compiler\n" >&2 printf " MODULE_FILE=${MODULE_FILE}\n" >&2 printf " PLATFORM=${PLATFORM}\n" >&2 @@ -226,8 +232,8 @@ CMAKE_SETTINGS="${CMAKE_SETTINGS} -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" if [ ! -z "${APPLICATION}" ]; then CMAKE_SETTINGS="${CMAKE_SETTINGS} -DAPP=${APPLICATION}" fi -if [ ! -z "${CCPP}" ]; then - CMAKE_SETTINGS="${CMAKE_SETTINGS} -DCCPP=${CCPP}" +if [ ! -z "${CCPP_SUITES}" ]; then + CMAKE_SETTINGS="${CMAKE_SETTINGS} -DCCPP_SUITES=${CCPP_SUITES}" fi if [ ! -z "${ENABLE_OPTIONS}" ]; then CMAKE_SETTINGS="${CMAKE_SETTINGS} -DENABLE_OPTIONS=${ENABLE_OPTIONS}" @@ -243,17 +249,17 @@ if [ "${VERBOSE}" = true ]; then fi # Before we go on load modules, we first need to activate Lmod for some systems -source ${SRC_DIR}/etc/lmod-setup.sh +source ${SRW_DIR}/etc/lmod-setup.sh $MACHINE # source the module file for this platform/compiler combination, then build the code printf "... Load MODULE_FILE and create BUILD directory ...\n" -module use ${SRC_DIR}/modulefiles +module use ${SRW_DIR}/modulefiles module load ${MODULE_FILE} module list mkdir -p ${BUILD_DIR} cd ${BUILD_DIR} printf "... Generate CMAKE configuration ...\n" -cmake ${SRC_DIR} ${CMAKE_SETTINGS} 2>&1 | tee log.cmake +cmake ${SRW_DIR} ${CMAKE_SETTINGS} 2>&1 | tee log.cmake printf "... Compile executables ...\n" make ${MAKE_SETTINGS} 2>&1 | tee log.make diff --git a/docs/INSTALL b/docs/INSTALL index 4b659bb1d1..e53044f6ad 100644 --- a/docs/INSTALL +++ b/docs/INSTALL @@ -42,7 +42,7 @@ source etc/lmod-setup.sh PLATFORM # From here on, we can assume Lmod is loaded and ready to go. Then we load the specific # module for a given PLATFORM and COMPILER as follows -module use modulefiles +module use $PWD/modulefiles #full path to modulefiles directory module load build_[PLATFORM]_[COMPILER] # Supported CMake flags: diff --git a/docs/UsersGuide/source/BuildRunSRW.rst b/docs/UsersGuide/source/BuildRunSRW.rst index 629210ec27..94e7deda10 100644 --- a/docs/UsersGuide/source/BuildRunSRW.rst +++ b/docs/UsersGuide/source/BuildRunSRW.rst @@ -81,7 +81,11 @@ The SRW Application source code is publicly available on GitHub. To download the COMMENT: This will need to be changed to the updated release branch of the SRW repo once it exists. The cloned repository contains the configuration files and sub-directories shown in -:numref:`Table %s `. +:numref:`Table %s `. The user may set an ``$SRW`` environmental variable to point to the location of the new ``ufs-srweather-app`` repository. For example, if ``ufs-srweather-app`` was cloned into the $HOME directory: + +.. code-block:: console + + export SRW=$HOME/ufs-srweather-app .. _FilesAndSubDirs: @@ -129,11 +133,12 @@ Run the executable that pulls in SRW App components from external repositories: .. code-block:: console - cd ufs-srweather-app + cd $SRW ./manage_externals/checkout_externals The script should output dialogue indicating that it is retrieving different code repositories. It may take several minutes to download these repositories. + .. _BuildExecutables: Set Up the Environment and Build the Executables @@ -141,6 +146,7 @@ Set Up the Environment and Build the Executables .. _DevBuild: + ``devbuild.sh`` Approach ----------------------------- @@ -152,7 +158,7 @@ On Level 1 systems for which a modulefile is provided under the ``modulefiles`` where ```` is replaced with the name of the platform the user is working on. Valid values are: ``cheyenne`` | ``gaea`` | ``hera`` | ``jet`` | ``macos`` | ``odin`` | ``orion`` | ``singularity`` | ``wcoss_dell_p3`` -If compiler auto-detection fails for some reason, specify it using the ``--compiler`` argument. FOr example: +If compiler auto-detection fails for some reason, specify it using the ``--compiler`` argument. For example: .. code-block:: console @@ -232,6 +238,7 @@ Set Up the Workflow Environment .. attention:: If users successfully built the executables in :numref:`Step %s `, they should skip to step :numref:`Step %s `. + For the CMake steps on MacOS systems, follow the approach in :numref:`Step %s `. If the ``devbuild.sh`` approach failed, users need to set up their environment to run a workflow on their specific platform. First, users should make sure ``Lmod`` is the app used for loading modulefiles. This is the case on most Level 1 systems; however, on systems such as Gaea/Odin, the default modulefile loader is from Cray and must be switched to Lmod. For example, on Gaea, assuming a ``bash`` login shell, run: @@ -288,6 +295,65 @@ The build will take a few minutes to complete. When it starts, a random number i If you see the build.out file, but there is no ``ufs-srweather-app/bin`` directory, wait a few more minutes for the build to complete. +.. _MacDetails: + +Additional Details for Building on MacOS +------------------------------------------ + +.. note:: + Users not building the SRW App to run on MacOS may skip to the :ref:`next section `. + +The SRW App can be built on MacOS systems, presuming HPC-Stack has already been successfully installed. The following two options have been tested: + +* **Option 1:** MacBookAir 2020, M1 chip (arm64, running natively), 4+4 cores, Big Sur 11.6.4, GNU compiler suite v.11.2.0_3 (gcc, gfortran, g++); no MPI pre-installed + +* **Option 2:** MacBook Pro 2015, 2.8 GHz Quad-Core Intel Core i7 (x86_64), Catalina OS X 10.15.7, GNU compiler suite v.11.2.0_3 (gcc, gfortran, g++); no MPI pre-installed + +The ``build_macos_gnu`` modulefile initializes the module environment, lists the location of HPC-Stack modules, loads the meta-modules and modules, and sets compilers, additional flags, and environment variables needed for building the SRW App. The modulefile must be modified to include the absolute path to the user's HPC-Stack installation and ``ufs-srweather-app`` directories. In particular, the following section must be modified: + +.. code-block:: console + + # This path should point to your HPCstack installation directory + setenv HPCstack "/Users/username/hpc-stack/install" + + # This path should point to your SRW Application directory + setenv SRW "/Users/username/ufs-srweather-app" + +An excerpt of the ``build_macos_gnu`` contents appears below for Option 1. To use Option 2, the user will need to comment out the lines specific to Option 1 and uncomment the lines specific to Option 2 in the ``build_macos_gnu`` modulefile. Additionally, users need to verify that all file paths reflect their system's configuration and that the correct version numbers for software libraries appear in the modulefile. + +.. code-block:: console + + # Option 1 compiler paths: + setenv CC "/opt/homebrew/bin/gcc" + setenv FC "/opt/homebrew/bin/gfortran" + setenv CXX "/opt/homebrew/bin/g++" + + # Option 2 compiler paths: + #setenv CC "/usr/local/bin/gcc" + #setenv FC "/usr/local/bin/gfortran" + #setenv CXX "/usr/local/bin/g++" + +Then, users must source the Lmod setup file, just as they would on other systems, and load the modulefiles needed for building and running SRW App: + +.. code-block:: console + + source etc/lmod-setup.sh macos + module use + module load build_macos_gnu + export LDFLAGS="-L${MPI_ROOT}/lib" + +In a csh/tcsh shell, users would run ``source etc/lmod-setup.csh macos`` in place of the first line in the code above. + +.. note:: + If you execute ``source etc/lmod-setup.sh`` on systems that don't need it, it will simply do a ``module purge``. + +Additionally, for Option 1 systems, set the variable ``ENABLE_QUAD_PRECISION`` to ``OFF`` in ``$SRW/src/ufs-weather-model/FV3/atmos_cubed_sphere/CMakeLists.txt`` file. This change is optional if using Option 2 to build the SRW App. You could use a streamline editor `sed` to change it: + +.. code-block:: console + + sed -i .bak 's/QUAD_PRECISION\" ON)/QUAD_PRECISION\" OFF)/' $SRW/src/ufs-weather-model/FV3/atmos_cubed_sphere/CMakeLists.txt + +Proceed to building executables using CMake in :numref:`Step %s ` .. _Data: @@ -387,7 +453,7 @@ settings. There is usually no need for a user to modify the default configuratio +----------------------+------------------------------------------------------------+ | NOMADS | NOMADS, NOMADS_file_type | +----------------------+------------------------------------------------------------+ - | External model | USE_USER_STAGED_EXTRN_FILES, EXTRN_MDL_SOURCE_BASEDRI_ICS, | + | External model | USE_USER_STAGED_EXTRN_FILES, EXTRN_MDL_SOURCE_BASEDIR_ICS, | | | EXTRN_MDL_FILES_ICS, EXTRN_MDL_SOURCE_BASEDIR_LBCS, | | | EXTRN_MDL_FILES_LBCS | +----------------------+------------------------------------------------------------+ @@ -561,14 +627,18 @@ To get started, make a copy of ``config.community.sh``. From the ``ufs-srweather .. code-block:: console - cd regional_workflow/ush + cd $SRW/regional_workflow/ush cp config.community.sh config.sh The default settings in this file include a predefined 25-km :term:`CONUS` grid (RRFS_CONUS_25km), the :term:`GFS` v16 physics suite (FV3_GFS_v16 :term:`CCPP`), and :term:`FV3`-based GFS raw external model data for initialization. Next, edit the new ``config.sh`` file to customize it for your machine. At a minimum, change the ``MACHINE`` and ``ACCOUNT`` variables; then choose a name for the experiment directory by setting ``EXPT_SUBDIR``. If you have pre-staged the initialization data for the experiment, set ``USE_USER_STAGED_EXTRN_FILES="TRUE"``, and set the paths to the data for ``EXTRN_MDL_SOURCE_BASEDIR_ICS`` and ``EXTRN_MDL_SOURCE_BASEDIR_LBCS``. -Sample settings are indicated below for Level 1 platforms. Detailed guidance applicable to all systems can be found in :numref:`Chapter %s: Configuring the Workflow `, which discusses each variable and the options available. Additionally, information about the four predefined Limited Area Model (LAM) Grid options can be found in :numref:`Chapter %s: Limited Area Model (LAM) Grids `. +.. note:: + + MacOS users should refer to :numref:`Section %s ` for details on configuring an experiment on MacOS. + +Sample settings are indicated below for Level 1 platforms. Detailed guidance applicable to all systems can be found in :numref:`Chapter %s: Configuring the Workflow `, which discusses each variable and the options available. Additionally, information about the three predefined Limited Area Model (LAM) Grid options can be found in :numref:`Chapter %s: Limited Area Model (LAM) Grids `. .. important:: @@ -659,8 +729,7 @@ For WCOSS_DELL_P3: .. note:: The values of the configuration variables should be consistent with those in the - ``valid_param_vals script``. In addition, various example configuration files can be - found in the ``regional_workflow/tests/baseline_configs`` directory. + ``valid_param_vals script``. In addition, various example configuration files can be found in the ``regional_workflow/tests/baseline_configs`` directory. .. _VXConfig: @@ -715,8 +784,8 @@ These tasks are independent, so users may set some values to "TRUE" and others t .. _SetUpPythonEnv: -Set up the Python and other Environment Parameters --------------------------------------------------- +Set Up the Python and Other Environment Parameters +---------------------------------------------------- The workflow requires Python 3 with the packages 'PyYAML', 'Jinja2', and 'f90nml' available. This Python environment has already been set up on Level 1 platforms, and it can be activated in the following way (from ``/ufs-srweather-app/regional_workflow/ush``): .. code-block:: console @@ -733,6 +802,142 @@ This command will activate the ``regional_workflow`` conda environment. The user source ~/.bashrc conda activate regional_workflow +.. _MacConfig: + +Configuring an Experiment on MacOS +------------------------------------------------------------ + +In principle, the configuration process for MacOS systems is the same as for other systems. However, the details of the configuration process on MacOS require a few extra steps. + +.. note:: + Examples in this subsection presume that the user is running Terminal.app with a bash shell environment. If this is not the case, users will need to adjust the commands to fit their command line application and shell environment. + +.. _MacMorePackages: + +Install Additional Packages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Check the version of bash, and upgrade it if it is lower than 4. Additionally, install the ``coreutils`` package: + +.. code-block:: console + + bash --version + brew upgrade bash + brew install coreutils + +.. _MacVEnv: + +Create a Python Virtual Environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Users must create a python virtual environment for running the SRW on MacOS. This involves setting python3 as default, adding required python modules, and sourcing the ``regional_workflow``. + +.. code-block:: console + + python3 -m pip --version + python3 -m pip install --upgrade pip + python3 -m ensurepip --default-pip + python3 -m venv $HOME/venv/regional_workflow + source $HOME/venv/regional_workflow/bin/activate + python3 -m pip install jinja2 + python3 -m pip install pyyaml + python3 -m pip install f90nml + python3 -m pip install ruby OR: brew install ruby + +The virtual environment can be deactivated by running the ``deactivate`` command. The virtual environment built here will be reactivated in :numref:`Step %s ` and needs to be used to generate the workflow and run the experiment. + +Install Rocoto +^^^^^^^^^^^^^^^^^^ + +.. note:: + Users may `install Rocoto `__ if they want to make use of a workflow manager to run their experiments. However, this option has not been tested yet on MacOS and is not supported for this release. + + +Configure the SRW App +^^^^^^^^^^^^^^^^^^^^^^^^ + +Users will need to configure their experiment just like on any other system. From the ``$SRW/regional_workflow/ush`` directory, users can copy the settings from ``config.community.sh`` into a ``config.sh`` file (see :numref:`Section %s `) above. In the ``config.sh`` file, users should set ``MACHINE="macos"`` and modify additional variables as needed. For example: + +.. code-block:: console + + MACHINE="macos" + ACCOUNT="user" + EXPT_SUBDIR="" + COMPILER="gnu" + VERBOSE="TRUE" + RUN_ENVIR="community" + PREEXISTING_DIR_METHOD="rename" + + PREDEF_GRID_NAME="RRFS_CONUS_25km" + QUILTING="TRUE" + +Due to the limited number of processors on Mac OS systems, users must configure the domain decomposition defaults (usually, there are only 8 CPUs in M1-family chips and 4 CPUs for x86_64). + +For :ref:`Option 1 `, add the following information to ``config.sh``: + +.. code-block:: console + + LAYOUT_X="${LAYOUT_X:-3}" + LAYOUT_Y="${LAYOUT_Y:-2}" + WRTCMP_write_groups="1" + WRTCMP_write_tasks_per_group="2" + +For :ref:`Option 2 `, add the following information to ``config.sh``: + +.. code-block:: console + + LAYOUT_X="${LAYOUT_X:-3}" + LAYOUT_Y="${LAYOUT_Y:-1}" + WRTCMP_write_groups="1" + WRTCMP_write_tasks_per_group="1" + +Configure the Machine File +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Configure the machine file based on the number of CPUs in the system (8 or 4). Specify the following variables in ``$SRW/regional_workflow/ush/machine/macos.sh``: + +For Option 1 (8 CPUs): + +.. code-block:: console + + # Commands to run at the start of each workflow task. + PRE_TASK_CMDS='{ ulimit -a; }' + + # Architecture information + WORKFLOW_MANAGER="none" + NCORES_PER_NODE=${NCORES_PER_NODE:-8} + SCHED=${SCHED:-"none"} + + # UFS SRW App specific paths + FIXgsm="path/to/FIXgsm/files" + FIXaer="path/to/FIXaer/files" + FIXlut="path/to/FIXlut/files" + TOPO_DIR="path/to/FIXgsm/files" # (path to location of static input files + used by the ``make_orog`` task) + SFC_CLIMO_INPUT_DIR="path/to/FIXgsm/files" # (path to location of static surface climatology + input fields used by ``sfc_climo_gen``) + + # Run commands for executables + RUN_CMD_SERIAL="time" + RUN_CMD_UTILS="mpirun -np 4" + RUN_CMD_FCST='mpirun -np ${PE_MEMBER01}' + RUN_CMD_POST="mpirun -np 4" + +Using Option 2 with 4 CPUs requires ``NCORES_PER_NODE=${NCORES_PER_NODE:-4}`` in the above example. + +.. _MacActivateWFenv: + +Activate the Workflow Environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``regional_workflow`` environment can be activated on MacOS as it is for any other system: + +.. code-block:: console + + cd $SRW/regional_workflow/ush + module load wflow_macos + +This should activate the ``regional_workflow`` environment created in :numref:`Step %s `. From here, the user may continue to the :ref:`next step ` and generate the regional workflow. + .. _GenerateWorkflow: @@ -941,7 +1146,7 @@ If the login shell is csh/tcsh, it can be set using: .. code-block:: console - setenv EXPTDIR /path-to-experiment/directory + setenv EXPTDIR // Launch the Rocoto Workflow Using a Script @@ -1104,6 +1309,7 @@ After finishing the experiment, open the crontab using ``crontab -e`` and delete On Orion, *cron* is only available on the orion-login-1 node, so users will need to work on that node when running *cron* jobs on Orion. + The workflow run is complete when all tasks have "SUCCEEDED", and the rocotostat command outputs a table similar to the one :ref:`above `. .. _PlotOutput: diff --git a/docs/UsersGuide/source/Introduction.rst b/docs/UsersGuide/source/Introduction.rst index 9e1e86b3dc..208162a9e2 100644 --- a/docs/UsersGuide/source/Introduction.rst +++ b/docs/UsersGuide/source/Introduction.rst @@ -14,9 +14,6 @@ The SRW App v1.0.0 citation is as follows and should be used when presenting res UFS Development Team. (2021, March 4). Unified Forecast System (UFS) Short-Range Weather (SRW) Application (Version v1.0.0). Zenodo. https://doi.org/10.5281/zenodo.4534994 -.. - COMMENT: Update version numbers/citation for release! Also update release date for citation! - How to Use This Document ======================== @@ -51,7 +48,7 @@ The instructions in this documentation assume that users have certain background * Familiarity with LINUX/UNIX systems * Command line basics * System configuration knowledge (e.g., compilers, environment variables, paths, etc.) -* Numerical Weather Prediction +* Numerical Weather Prediction (concepts of parameterizations: physical, microphysical, convective) * Meteorology (particularly meteorology at the scales being predicted) .. @@ -73,12 +70,14 @@ The UFS SRW Application has been designed so that any sufficiently up-to-date ma * POSIX-compliant UNIX-style operating system -* >40 GB disk space +* >82 GB disk space + + * 53 GB input data for a standard collection of global database, or "fix" data (topography, climatology, observational database), and boundary conditions for a short 12-h test forecast on CONUS 25km domain. See data download instructions in :numref:`Step %s `. + * 8 GB for :term:`HPC-Stack` full installation + * 3 GB for ufs-srweather-app installation + * 1 GB boundary conditions for a short 12-h test forecast on CONUS 25km domain. See data download instructions in :numref:`Step %s ` + * 17 GB for a 12-h test forecast on CONUS 25km domain, with model output saved hourly, see :numref:`Step %s ` - * 18 GB input data from GFS, RAP, and HRRR for "out-of-the-box" SRW App case described in :numref:`Chapter %s ` - * 6 GB for :term:`HPC-Stack` full installation - * 1 GB for ufs-srweather-app installation - * 11 GB for 48hr forecast on CONUS 25km domain * 4GB memory (CONUS 25km domain) @@ -88,7 +87,7 @@ The UFS SRW Application has been designed so that any sufficiently up-to-date ma * C and C++ compilers compatible with the Fortran compiler - * gcc v9+, ifort v18+, and clang v9+ (macOS, native Apple clang or LLVM clang) have been tested + * gcc v9+, ifort v18+, and clang v9+ (macOS, native Apple clang, LLVM clang, GNU) have been tested * Python v3.6+, including prerequisite packages ``jinja2``, ``pyyaml`` and ``f90nml`` @@ -96,41 +95,37 @@ The UFS SRW Application has been designed so that any sufficiently up-to-date ma * Perl 5 -* git v1.8+ +* git v2.12+ -.. - COMMENT: Should curl/wget/TIFF library also be required? These are listed as prerequisites for building HPC-Stack on generic MacOS/Linux +* curl + +* wget +.. + COMMENT: Should curl/wget/TIFF library also be required? These are listed as prerequisites for building HPC-Stack on generic MacOS/Linux | A: TIFF could be installed with the HPC-stack build; others are needed The following software is also required to run the SRW Application, but the :term:`HPC-Stack` (which contains the software libraries necessary for building and running the SRW App) can be configured to build these requirements: -* CMake v3.15+ +* CMake v3.20+ * MPI (MPICH, OpenMPI, or other implementation) - * Only **MPICH** can be built with HPC-Stack. Other options must be installed separately by the user. - -* Software libraries - - * netCDF (C and Fortran libraries) - * HDF5 - * ESMF 8.2.0 - * Jasper - * libJPG - * libPNG - * zlib + * Only **MPICH** or **OpenMPI** can be built with HPC-Stack. Other implementations must be installed separately by the user. For MacOS systems, some additional software is needed. It is recommended that users install this software using the `Homebrew `__ package manager for MacOS: +* brew install bash + + * OR: brew upgrade bash * brew install gcc@11 * brew install cmake * brew install make * brew install wget +* brew install lmod * brew install coreutils -* brew install pkg-config .. - COMMENT: Is this still accurate? It seems like we should delete the last one. And gcc@11 is basically the same as requiring fortran/C/C++ compilers, no? CMake is listed above. + COMMENT: Is this still accurate? It seems like we should delete the last one. And gcc@11 is basically the same as requiring fortran/C/C++ compilers, no? CMake is listed above. | A: updated bash requirements Optional but recommended prerequisites for all systems: @@ -138,8 +133,9 @@ Optional but recommended prerequisites for all systems: * Bash v4+ * Rocoto Workflow Management System (1.3.1) * Python packages ``scipy``, ``matplotlib``, ``pygrib``, ``cartopy``, and ``pillow`` for graphics -* Lmod +.. + COMMENT: Lmod is listed as required .. _ComponentsOverview: diff --git a/docs/UsersGuide/source/Quickstart.rst b/docs/UsersGuide/source/Quickstart.rst index 64685d8a02..3e715a998f 100644 --- a/docs/UsersGuide/source/Quickstart.rst +++ b/docs/UsersGuide/source/Quickstart.rst @@ -334,6 +334,12 @@ Check the batch script output file in your experiment directory for a “SUCCESS | | | | forecast hour) | +------------+------------------------+----------------+----------------------------+ +Users can access log files for specific tasks in the ``$EXPTDIR/log`` directory. To see how the experiment is progressing, users can also check the end of the ``log.launch_FV3LAM_wflow`` file from the command line: + +.. code-block:: console + + tail -n 40 log.launch_FV3LAM_wflow + .. hint:: If any of the scripts return an error that "Primary job terminated normally, but one process returned a non-zero exit code," there may not be enough space on one node to run the process. On an HPC system, the user will need to allocate a(nother) compute node. The process for doing so is system-dependent, and users should check the documentation available for their HPC system. Instructions for allocating a compute node on NOAA Cloud systems can be viewed in the :numref:`Step %s ` as an example. diff --git a/docs/UsersGuide/source/WE2Etests.rst b/docs/UsersGuide/source/WE2Etests.rst index cb4b0a8673..1c50a1f0e2 100644 --- a/docs/UsersGuide/source/WE2Etests.rst +++ b/docs/UsersGuide/source/WE2Etests.rst @@ -162,6 +162,7 @@ The full usage statement for ``run_WE2E_tests.sh`` is as follows: [use_cron_to_relaunch="..."] \ [cron_relaunch_intvl_mnts="..."] \ [verbose="..."] \ + [generate_csv_file="..."] \ [machine_file="..."] \ [stmp="..."] \ [ptmp="..."] \ @@ -183,7 +184,8 @@ in the directory ``ufs-srweather-app/regional_workflow/tests/WE2E``. The WE2E Test Information File ================================ In addition to creating the WE2E tests' experiment directories and optionally creating -cron jobs to launch their workflows, the ``run_WE2E_tests.sh`` script generates (if necessary) +cron jobs to launch their workflows, the ``run_WE2E_tests.sh`` script generates (if +necessary and if not explicitly disabled by the ``generate_csv_file="FALSE"`` flag to this script) a CSV (Comma-Separated Value) file named ``WE2E_test_info.csv`` that contains information on the full set of WE2E tests. This file serves as a single location where relevant information about the WE2E tests can be found. It can be imported into Google Sheets @@ -191,27 +193,61 @@ using the "|" (pipe symbol) character as the custom field separator. The rows of file/sheet represent the full set of available tests (not just the ones to be run), while the columns contain the following information (column titles are included in the CSV file): -| Column 1: -| The primary test name and (in parentheses) the category subdirectory it is - located in. +| **Column 1** +| The primary test name followed by the category subdirectory it is + located in (the latter in parentheses). -| Column 2: -| Any alternate names for the test (if any) followed by their category subdirectories +| **Column 2** +| Any alternate names for the test followed by their category subdirectories (in parentheses). -| Column 3: +| **Column 3** | The test description. -| Column 4: -| The number of times the forecast model will be run by the test. This gives an idea - of how expensive the test is. It is calculated using quantities such as the number - of cycle dates (i.e. forecast model start dates) and the number of of ensemble members - (if running ensemble forecasts). The are in turn obtained directly or indirectly - from the quantities in Columns 5, 6, .... +| **Column 4** +| The relative cost of running the dynamics in the test. This gives an + idea of how expensive the test is relative to a reference test that runs + a single 6-hour forecast on the ``RRFS_CONUS_25km`` predefined grid using + its default time step. To calculate the relative cost, the absolute cost + ``abs_cost`` is first calculated as follows: -| Columns 5,6,...: -| The values of various experiment variables (if defined) in each test's configuration - file. Currently, the following experiment variables are included: +.. code-block:: + + abs_cost = nx*ny*num_time_steps*num_fcsts + +| Here, ``nx`` and ``ny`` are the number of grid points in the horizontal + (``x`` and ``y``) directions, ``num_time_steps`` is the number of time + steps in one forecast, and ``num_fcsts`` is the number of forecasts the + test runs (see Column 5 below). [Note that this cost calculation does + not (yet) differentiate between different physics suites.] The relative + cost ``rel_cost`` is then calculated using + +.. code-block:: + + rel_cost = abs_cost/abs_cost_ref + +| where ``abs_cost_ref`` is the absolute cost of running the reference forecast + described above, i.e. a single (``num_fcsts = 1``) 6-hour forecast + (``FCST_LEN_HRS = 6``) on the ``RRFS_CONUS_25km grid`` (which currently has + ``nx = 219``, ``ny = 131``, and ``DT_ATMOS = 40 sec`` (so that ``num_time_steps + = FCST_LEN_HRS*3600/DT_ATMOS = 6*3600/40 = 540``), i.e. + +.. code-block:: + + abs_cost_ref = 219*131*540*1 = 15,492,060 + +| **Column 5** +| The number of times the forecast model will be run by the test. This + is calculated using quantities such as the number of cycle dates (i.e. + forecast model start dates) and the number of of ensemble members (which + is greater than 1 if running ensemble forecasts and 1 otherwise). The + latter are in turn obtained directly or indirectly from the quantities + in Columns 6, 7, .... + +| **Columns 6, 7, ...** +| The values of various experiment variables (if defined) in each test's + configuration file. Currently, the following experiment variables are + included: | ``PREDEF_GRID_NAME`` | ``CCPP_PHYS_SUITE`` @@ -222,18 +258,27 @@ while the columns contain the following information (column titles are included | ``CYCL_HRS`` | ``INCR_CYCL_FREQ`` | ``FCST_LEN_HRS`` + | ``DT_ATMOS`` | ``LBC_SPEC_INTVL_HRS`` | ``NUM_ENS_MEMBERS`` -Additional fields (columns) will likely be added to the CSV file in the near future. +Additional fields (columns) may be added to the CSV file in the future. Note that the CSV file is not part of the ``regional_workflow`` repo (i.e. it is not tracked by the repo). The ``run_WE2E_tests.sh`` script will generate a CSV -file if (1) the CSV file doesn't already exist, or (2) the CSV file does exist -but changes have been made to one or more of the category subdirectories (e.g., -test configuration files modified, added, or deleted) since the creation of the -CSV file. Thus, ``run_WE2E_tests.sh`` will always create a CSV file the first -time it is run in a fresh git clone of the SRW App. +file if the ``generate_csv_file`` flag to this script has not explicitly been +set to ``"FALSE"`` and if either one of the following is true: + +#. The CSV file doesn't already exist. +#. The CSV file does exist, but changes have been made to one or more of the + category subdirectories (e.g., test configuration files modified, added, + or deleted) since the creation of the CSV file. + +Thus, unless the ``generate_csv_file`` flag is set to ``"FALSE"``, the +``run_WE2E_tests.sh`` will create a CSV file the first time it is run in a +fresh git clone of the SRW App. (The ``generate_csv_file`` flag is provided +because the CSV file generation can be slow, so users may wish to skip this +step since it is not a necessary part of running the tests.) Checking Test Status @@ -327,6 +372,7 @@ above, such as ``wflow_features``: Adding a New WE2E Test Category ----------------------------------- + To create a new test category called, e.g., ``new_category``: #. In the directory ``ufs-srweather-app/regional_workflow/tests/WE2E/test_configs``, create a new directory named ``new_category``. @@ -372,9 +418,14 @@ In this situation, the primary name for the test is ``grid_RRFS_CONUScompact_25k Note the following: -* A primary test can have more than one alternate test name (by having more than one symlink point to the test's configuration file). -* The symlinks representing the alternate test names can be in the same or a different category directory. -* The --relative flag makes the symlink relative (i.e., within/below the ``regional_workflow`` directory structure) so that it stays valid when copied to other locations. However, the ``--relative`` flag may be different and/or not exist on every platform. +* A primary test can have more than one alternate test name (by having + more than one symlink point to the test's configuration file). +* The symlinks representing the alternate test names can be in the same + or a different category directory. +* The ``--relative`` flag makes the symlink relative (i.e., within/below + the ``regional_workflow`` directory structure) so that it stays valid + when copied to other locations. (Note however that this flag is + platform-dependent and may not exist on some platform.) * To determine whether a test has one or more alternate names, a user can view the CSV file ``WE2E_test_info.csv`` that ``run_WE2E_tests.sh`` generates. Recall from :numref:`Section %s ` that column 1 of this CSV @@ -382,8 +433,9 @@ Note the following: any alternate names (and their categories). * With this primary/alternate test naming convention, a user can list either the primary test name or one of the alternate test names in the experiments list file - (e.g. ``my_tests.txt``) that ``run_WE2E_tests.sh`` reads in. If both primary and - one or more alternate test names are listed, then ``run_WE2E_tests.sh`` will exit - with a warning message without running any tests. + (e.g. ``my_tests.txt``) read in by ``run_WE2E_tests.sh``. If more than one name + is listed for the same test (e.g. the primary name and and an alternate name, + two alternate names, etc), ``run_WE2E_tests.sh`` will exit with a warning message + without running any tests. diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index 1486fffccf..a6e1e3a5a4 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -1,7 +1,14 @@ #!/bin/csh if ( $# == 0 ) then - set L_MACHINE=${MACHINE} + cat << EOF_USAGE +Usage: source etc/lmod-setup.csh PLATFORM + +OPTIONS: + PLATFORM - name of machine you are building on + (e.g. cheyenne | hera | jet | orion | wcoss_dell_p3) +EOF_USAGE + exit 1 else set L_MACHINE=$1 endif diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index 15bb24fc87..d8f59ae267 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -2,6 +2,14 @@ if [ $# = 0 ]; then L_MACHINE=${MACHINE} + cat << EOF_USAGE +Usage: source etc/lmod-setup.sh PLATFORM + +OPTIONS: + PLATFORM - name of machine you are building on + (e.g. cheyenne | hera | jet | orion | wcoss_dell_p3) +EOF_USAGE + exit 1 else L_MACHINE=$1 fi @@ -43,4 +51,3 @@ elif [ "$L_MACHINE" = odin ]; then else module purge fi - diff --git a/modulefiles/build_cheyenne_gnu b/modulefiles/build_cheyenne_gnu index add5ee3b0f..f8be4be1ea 100644 --- a/modulefiles/build_cheyenne_gnu +++ b/modulefiles/build_cheyenne_gnu @@ -9,22 +9,23 @@ module-whatis "Loads libraries needed for building SRW on Cheyenne" module load cmake/3.22.0 module load ncarenv/1.3 -module load gnu/10.1.0 -module load mpt/2.22 +module load gnu/11.2.0 +module load mpt/2.25 module load ncarcompilers/0.5.0 module load python/3.7.9 module unload netcdf -module use /glade/p/ral/jntp/GMTB/tools/hpc-stack-v1.2.0/modulefiles/stack +module use /glade/work/epicufsrt/GMTB/tools/gnu/11.2.0/hpc-stack-v1.2.0/modulefiles/stack module load hpc/1.2.0 -module load hpc-gnu/10.1.0 -module load hpc-mpt/2.22 +module load hpc-gnu +module load hpc-mpt/2.25 module load srw_common module load g2/3.4.3 -module load esmf/8_2_0 +module load w3emc/2.9.2 +module load esmf/8.3.0b09 +module load mapl/2.11.0-esmf-8.3.0b09 module load netcdf/4.7.4 -module load png/1.6.35 module load pio/2.5.2 setenv CMAKE_C_COMPILER mpicc diff --git a/modulefiles/build_cheyenne_intel b/modulefiles/build_cheyenne_intel index b83c36a5df..c1d868d458 100644 --- a/modulefiles/build_cheyenne_intel +++ b/modulefiles/build_cheyenne_intel @@ -2,33 +2,32 @@ proc ModulesHelp { } { puts stderr "This module loads libraries for building SRW on" - puts stderr "the CISL Cheyenne machine using Intel-19.1.1" + puts stderr "the CISL Cheyenne machine using Intel-2022.1" } module-whatis "Loads libraries needed for building SRW on Cheyenne" +module purge module load cmake/3.22.0 -module load ncarenv/1.3 -module load intel/2021.2 -module load mpt/2.22 -module load ncarcompilers/0.5.0 +module load intel/2022.1 +module load mpt/2.25 module load python/3.7.9 -module unload netcdf -module use /glade/p/ral/jntp/GMTB/tools/hpc-stack-v1.2.0/modulefiles/stack +module use /glade/work/epicufsrt/GMTB/tools/intel/2022.1/hpc-stack-v1.2.0_6eb6/modulefiles/stack module load hpc/1.2.0 -module load hpc-intel/2021.2 -module load hpc-mpt/2.22 +module load hpc-intel/2022.1 +module load hpc-mpt/2.25 module load srw_common -module load g2/3.4.3 -module load esmf/8_2_0 +module load g2/3.4.5 +module load esmf/8.3.0b09 +module load mapl/2.11.0-esmf-8.3.0b09 module load netcdf/4.7.4 -module load png/1.6.35 -module load pio/2.5.2 +module load libpng/1.6.37 +module load pio/2.5.3 setenv CMAKE_C_COMPILER mpicc -setenv CMAKE_CXX_COMPILER mpicxx +setenv CMAKE_CXX_COMPILER mpixx setenv CMAKE_Fortran_COMPILER mpif90 setenv CMAKE_Platform cheyenne.intel diff --git a/modulefiles/build_gaea_intel b/modulefiles/build_gaea_intel index 15ed495a90..cce0e07468 100644 --- a/modulefiles/build_gaea_intel +++ b/modulefiles/build_gaea_intel @@ -14,6 +14,7 @@ module load srw_common module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles module load rocoto module load cmake/3.20.1 +module load png/1.6.35 setenv CC cc setenv FC ftn diff --git a/modulefiles/build_macos_gnu b/modulefiles/build_macos_gnu index 4e0c670b70..54a5d5db38 100644 --- a/modulefiles/build_macos_gnu +++ b/modulefiles/build_macos_gnu @@ -81,6 +81,7 @@ setenv CMAKE_Platform macos.gnu setenv CMAKE_Fortran_COMPILER_ID "GNU" setenv FFLAGS "-DNO_QUAD_PRECISION -fallow-argument-mismatch " + # export the environment variable LDFLAGS from the command line # after loading the current module: # export LDFLAGS="-L$MPI_ROOT/lib" diff --git a/modulefiles/build_wcoss2_intel b/modulefiles/build_wcoss2_intel new file mode 100644 index 0000000000..f7adefb4f7 --- /dev/null +++ b/modulefiles/build_wcoss2_intel @@ -0,0 +1,51 @@ +#%Module + +proc ModulesHelp { } { + puts stderr "This module loads libraries for building SRW on" + puts stderr "the NOAA WCOSS2 machine using Intel-19.1.3.304" +} + +module-whatis "Loads libraries needed for building SRW on WCOSS2 (Cactus/Dogwood)" + +module load envvar/1.0 + +module load PrgEnv-intel/8.1.0 +module load intel/19.1.3.304 +module load craype/2.7.13 +module load cray-mpich/8.1.7 + +module load cmake/3.20.2 + +setenv HPC_OPT /apps/ops/para/libs +module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304 +module use /apps/ops/para/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.7 + +module load jasper/2.0.25 +module load zlib/1.2.11 +module load libpng/1.6.37 +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.2 +module load esmf/8.3.0b09 +module load fms/2022.01 +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.5 +module load g2tmpl/1.10.0 +module load ip/3.3.3 +module load sp/2.3.3 +module load w3nco/2.4.1 + +module load libjpeg/9c +module load cray-pals/1.1.3 + +module load w3emc/2.9.2 +module load nemsio/2.5.2 +module load sigio/2.3.2 +module load sfcio/1.4.1 +module load wrf_io/1.2.0 + +setenv CMAKE_C_COMPILER cc +setenv CMAKE_CXX_COMPILER CC +setenv CMAKE_Fortran_COMPILER ftn +setenv CMAKE_Platform wcoss2 diff --git a/modulefiles/srw_common b/modulefiles/srw_common index c8c87f92a8..570a492468 100644 --- a/modulefiles/srw_common +++ b/modulefiles/srw_common @@ -2,14 +2,15 @@ module load jasper/2.0.25 module load zlib/1.2.11 -module load-any png/1.6.35 libpng/1.6.35 +module try-load png/1.6.35 +module try-load libpng/1.6.35 module load hdf5/1.10.6 module load-any netcdf/4.7.4 netcdf-c/4.7.4 module load-any netcdf/4.7.4 netcdf-fortran/4.5.4 module load-any pio/2.5.2 parallelio/2.5.2 -module load-any esmf/8_2_0 esmf/8.2.0 -module load fms/2021.03 +module load-any esmf/8.3.0b09 +module load fms module load bacio/2.4.1 module load crtm/2.3.0 @@ -22,7 +23,7 @@ module load upp/10.0.10 module load-any gftl-shared/v1.3.3 gftl-shared/1.3.3 module load-any yafyaml/v0.5.1 yafyaml/0.5.1 -module load-any mapl/2.11.0-esmf-8_2_0 mapl/2.11.0-esmf-8.2.0 +module load-any mapl/2.11.0-esmf-8.3.0b09 module load gfsio/1.4.1 module load landsfcutil/2.4.1 @@ -30,5 +31,5 @@ module load nemsio/2.5.2 module load nemsiogfs/2.5.3 module load sfcio/1.4.1 module load sigio/2.3.2 -module load w3emc/2.7.3 +module try-load w3emc/2.7.3 module load wgrib2/2.0.8 diff --git a/modulefiles/wflow_jet b/modulefiles/wflow_jet index 7aa7d0e67a..98e6843365 100644 --- a/modulefiles/wflow_jet +++ b/modulefiles/wflow_jet @@ -9,6 +9,7 @@ module-whatis "Loads libraries needed for running SRW on Jet" module load rocoto + module use /contrib/miniconda3/modulefiles module load miniconda3/4.5.12 diff --git a/test/build.sh b/test/build.sh index 83c30910e7..cb2119066d 100755 --- a/test/build.sh +++ b/test/build.sh @@ -21,7 +21,7 @@ function usage() { exit 1 } -machines=( hera jet cheyenne orion wcoss_cray wcoss_dell_p3 gaea odin singularity ) +machines=( hera jet cheyenne orion wcoss2 wcoss_dell_p3 gaea odin singularity macos noaacloud ) [[ $# -eq 2 ]] && usage