From d0c0c69c9cb46bc58d99f5b5e546b482118d79f8 Mon Sep 17 00:00:00 2001 From: Jesse McFarland Date: Wed, 25 May 2022 10:14:27 -0700 Subject: [PATCH 01/24] Fix artifact creation for branches with / in name (#275) The slash in branch names such as release/.* and feature/.* is interpreted as a directory separator. This change replaces the "/" character with a "_". --- .cicd/Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile index 158b2963ee..095eecb200 100644 --- a/.cicd/Jenkinsfile +++ b/.cicd/Jenkinsfile @@ -98,7 +98,8 @@ pipeline { } environment { - BUILD_VERSION = "${env.SRW_PLATFORM}-${env.SRW_COMPILER}-${env.BRANCH_NAME}-${env.BUILD_NUMBER}" + BRANCH_NAME_ESCAPED = env.BRANCH_NAME.replace('/', '_') + BUILD_VERSION = "${env.SRW_PLATFORM}-${env.SRW_COMPILER}-${env.BRANCH_NAME_ESCAPED}-${env.BUILD_NUMBER}" BUILD_NAME = "ufs-srweather-app_${env.BUILD_VERSION}" } From 3a2fd2bc278a55f778003ab514d7dcab4f3dc94f Mon Sep 17 00:00:00 2001 From: Gillian Petro <96886803+gspetro-NOAA@users.noreply.github.com> Date: Thu, 26 May 2022 14:23:50 -0400 Subject: [PATCH 02/24] Include documentation for building/running SRW App on Mac (#240) * updated docs * added git submodule * fix formatting * added new submodule commits * fixed ref links * finished Intro * finish Components & Intro edits * edited Rocoto workflow section of Quickstart * added minor hpc submodule commits * Updates to Rocoto Workflow in Quick Start * add to HPC-stack intro * submodule updates * added submodule docs edits * hpc-stack updates & formatting fixes * hpc-stack intro edits * bibtex attempted fix * add hpc-stack module edits * update sphinxcontrib version * add .readthedocs.yaml file * update .readthedocs.yaml file * update .readthedocs.yaml file * update conf.py * updates .readthedocs.yaml with submodules * updates .readthedocs.yaml with submodules * submodule updates * submodule updates * minor Intro edits * minor Intro edits * minor Intro edits * submodule updates * fixed typos in QS * QS updates * QS updates * QS updates * updates to InputOutput and QS * fix I/O doc typos * pull updates to hpc-stack docs * pull updates to hpc-stack docs * fix table wrapping * updates to QS for cloud * fix QS export statements * fix QS export statements * QS edits on bind, config * add bullet points to notes * running without rocoto * add HPC-Stack submodule w/docs * split QS into container/non-container approaches * added filepath changes for running in container on Orion, et al. * edits to overview and container QS * moved CodeReposAndDirs.rst info to the Introduction & deleted file * continued edits to SRWAppOverview * combine overview w/non-container docs * finish merging non-container guide & SRWOverview, rename/remove files, update FAQ * minor edits for Intro & QS * updates to BuildRun doc through 3.8.1 * edits to Build/Run and Components * remove .gitignore * fix Ch 3 title, 4 supported platform levels note * fix typos, add term links * other minor fixes/suggestions implemented * updated Intro based on feedback; changed SRW to SRW App throughout * update comment to Intro citation * add user-defined vertical levels to future work * Add instructions for srw_common module load * fix typo * update Intro & BuildRunSRW based on Mark's feedback * minor intro updates * 1st round of jwolff's edits * 2nd round of jwolff updates * update QS intro * fix minor physics details * update citation and physics suite name * add compute node allocation info to QS * add authoritative hpc-stack docs to Intro * create MacOS install/build instructions * add MacOS Build/Run instructions * fix MacOS Build/Run details * add MacOS info directly to Build/Run SRW chapter * minor details * minor edits * update Include-HPCInstall with mac installation docs * add note re: Terminal.app & bash shell in MacOS section * remove MacInstall file-contents added to BuildRunSRW * update hpc-stack submodule to include mac installation info * add MacOS config details * add MacOS config & run details * minor MacOS note * mention need to verify software library version #'s * update hpc-stack-mod * align MacDetails section with PR #238 info * remove gsed & alter related commands * update hpc-stack submodule * typos * switch from env to module load Co-authored-by: Will Mayfield <59745143+willmayfield@users.noreply.github.com> * Update BuildRunSRW.rst * update hpc-stack module docs & MacOS config.sh * update machine file instructions * updates to BuildRun chapter * fix typo Co-authored-by: gspetro Co-authored-by: Will Mayfield <59745143+willmayfield@users.noreply.github.com> --- docs/UsersGuide/source/BuildRunSRW.rst | 233 +++++++++++++++++++++++-- docs/UsersGuide/source/Quickstart.rst | 6 + docs/UsersGuide/source/WE2Etests.rst | 1 + 3 files changed, 229 insertions(+), 11 deletions(-) diff --git a/docs/UsersGuide/source/BuildRunSRW.rst b/docs/UsersGuide/source/BuildRunSRW.rst index 629210ec27..658ff72a34 100644 --- a/docs/UsersGuide/source/BuildRunSRW.rst +++ b/docs/UsersGuide/source/BuildRunSRW.rst @@ -81,7 +81,11 @@ The SRW Application source code is publicly available on GitHub. To download the COMMENT: This will need to be changed to the updated release branch of the SRW repo once it exists. The cloned repository contains the configuration files and sub-directories shown in -:numref:`Table %s `. +:numref:`Table %s `. The user may set an ``$SRW`` environmental variable to point to the location of the new ``ufs-srweather-app`` repository. For example, if ``ufs-srweather-app`` was cloned into the $HOME directory: + +.. code-block:: console + + export SRW=$HOME/ufs-srweather-app .. _FilesAndSubDirs: @@ -129,11 +133,12 @@ Run the executable that pulls in SRW App components from external repositories: .. code-block:: console - cd ufs-srweather-app + cd $SRW ./manage_externals/checkout_externals The script should output dialogue indicating that it is retrieving different code repositories. It may take several minutes to download these repositories. + .. _BuildExecutables: Set Up the Environment and Build the Executables @@ -141,6 +146,7 @@ Set Up the Environment and Build the Executables .. _DevBuild: + ``devbuild.sh`` Approach ----------------------------- @@ -152,7 +158,7 @@ On Level 1 systems for which a modulefile is provided under the ``modulefiles`` where ```` is replaced with the name of the platform the user is working on. Valid values are: ``cheyenne`` | ``gaea`` | ``hera`` | ``jet`` | ``macos`` | ``odin`` | ``orion`` | ``singularity`` | ``wcoss_dell_p3`` -If compiler auto-detection fails for some reason, specify it using the ``--compiler`` argument. FOr example: +If compiler auto-detection fails for some reason, specify it using the ``--compiler`` argument. For example: .. code-block:: console @@ -288,6 +294,70 @@ The build will take a few minutes to complete. When it starts, a random number i If you see the build.out file, but there is no ``ufs-srweather-app/bin`` directory, wait a few more minutes for the build to complete. +.. _MacDetails: + +Additional Details for Building on MacOS +------------------------------------------ + +.. note:: + Users not building the SRW App to run on MacOS may skip to the :ref:`next section `. + +The SRW App can be built on MacOS systems, presuming HPC-Stack has already been successfully installed. The following two options have been tested: + +* **Option 1:** MacBookAir 2020, M1 chip (arm64, running natively), 4+4 cores, Big Sur 11.6.4, GNU compiler suite v.11.2.0_3 (gcc, gfortran, g++); no MPI pre-installed + +* **Option 2:** MacBook Pro 2015, 2.8 GHz Quad-Core Intel Core i7 (x86_64), Catalina OS X 10.15.7, GNU compiler suite v.11.2.0_3 (gcc, gfortran, g++); no MPI pre-installed + +The ``build_macos_gnu`` modulefile initializes the module environment, lists the location of HPC-Stack modules, loads the meta-modules and modules, and sets compilers, additional flags, and environment variables needed for building the SRW App. The modulefile must be modified to include the absolute path to the user's HPC-Stack installation and ``ufs-srweather-app`` directories. In particular, the following section must be modified: + +.. code-block:: console + + # This path should point to your HPCstack installation directory + setenv HPCstack "/Users/username/hpc-stack/install" + + # This path should point to your SRW Application directory + setenv SRW "/Users/username/ufs-srweather-app" + +An excerpt of the ``build_macos_gnu`` contents appears below for Option 1. To use Option 2, the user will need to comment out the lines specific to Option 1 and uncomment the lines specific to Option 2 in the ``build_macos_gnu`` modulefile. Additionally, users need to verify that all file paths reflect their system's configuration and that the correct version numbers for software libraries appear in the modulefile. + +.. code-block:: console + + # Option 1 compiler paths: + setenv CC "/opt/homebrew/bin/gcc" + setenv FC "/opt/homebrew/bin/gfortran" + setenv CXX "/opt/homebrew/bin/g++" + + # Option 2 compiler paths: + #setenv CC "/usr/local/bin/gcc" + #setenv FC "/usr/local/bin/gfortran" + #setenv CXX "/usr/local/bin/g++" + +Then, users must source the Lmod setup file, just as they would on other systems, and load the modulefiles needed for building and running SRW App: + +.. code-block:: console + + source etc/lmod-setup.sh macos + module use + module load build_macos_gnu + +In a csh/tcsh shell, users would run ``source etc/lmod-setup.csh macos`` in place of the first line in the code above. + +.. note:: + If you execute ``source etc/lmod-setup.sh`` on systems that don't need it, it will simply do a ``module purge``. + +Additionally, for Option 1 systems, set the variable ``ENABLE_QUAD_PRECISION`` to ``OFF`` in line 35 of the ``$SRW/src/ufs-weather-model/FV3/atmos_cubed_sphere/CMakeLists.txt`` file. This change is optional if using Option 2 to build the SRW App. Using a text editor (e.g., vi, vim, emacs): + +.. code-block:: console + + option(ENABLE_QUAD_PRECISION "Enable compiler definition -DENABLE_QUAD_PRECISION" OFF) + +An alternative way to make this change is using a `sed` (streamline editor). From the command line, users can run one of two commands (user's preference): + +.. code-block:: console + + sed -i -e 's/QUAD_PRECISION\" ON)/QUAD_PRECISION\" OFF)/' CMakeLists.txt + sed -i -e 's/bin\/sh/bin\/bash/g' *sh + .. _Data: @@ -387,7 +457,7 @@ settings. There is usually no need for a user to modify the default configuratio +----------------------+------------------------------------------------------------+ | NOMADS | NOMADS, NOMADS_file_type | +----------------------+------------------------------------------------------------+ - | External model | USE_USER_STAGED_EXTRN_FILES, EXTRN_MDL_SOURCE_BASEDRI_ICS, | + | External model | USE_USER_STAGED_EXTRN_FILES, EXTRN_MDL_SOURCE_BASEDIR_ICS, | | | EXTRN_MDL_FILES_ICS, EXTRN_MDL_SOURCE_BASEDIR_LBCS, | | | EXTRN_MDL_FILES_LBCS | +----------------------+------------------------------------------------------------+ @@ -561,14 +631,18 @@ To get started, make a copy of ``config.community.sh``. From the ``ufs-srweather .. code-block:: console - cd regional_workflow/ush + cd $SRW/regional_workflow/ush cp config.community.sh config.sh The default settings in this file include a predefined 25-km :term:`CONUS` grid (RRFS_CONUS_25km), the :term:`GFS` v16 physics suite (FV3_GFS_v16 :term:`CCPP`), and :term:`FV3`-based GFS raw external model data for initialization. Next, edit the new ``config.sh`` file to customize it for your machine. At a minimum, change the ``MACHINE`` and ``ACCOUNT`` variables; then choose a name for the experiment directory by setting ``EXPT_SUBDIR``. If you have pre-staged the initialization data for the experiment, set ``USE_USER_STAGED_EXTRN_FILES="TRUE"``, and set the paths to the data for ``EXTRN_MDL_SOURCE_BASEDIR_ICS`` and ``EXTRN_MDL_SOURCE_BASEDIR_LBCS``. -Sample settings are indicated below for Level 1 platforms. Detailed guidance applicable to all systems can be found in :numref:`Chapter %s: Configuring the Workflow `, which discusses each variable and the options available. Additionally, information about the four predefined Limited Area Model (LAM) Grid options can be found in :numref:`Chapter %s: Limited Area Model (LAM) Grids `. +.. note:: + + MacOS users should refer to :numref:`Section %s ` for details on configuring an experiment on MacOS. + +Sample settings are indicated below for Level 1 platforms. Detailed guidance applicable to all systems can be found in :numref:`Chapter %s: Configuring the Workflow `, which discusses each variable and the options available. Additionally, information about the three predefined Limited Area Model (LAM) Grid options can be found in :numref:`Chapter %s: Limited Area Model (LAM) Grids `. .. important:: @@ -659,8 +733,7 @@ For WCOSS_DELL_P3: .. note:: The values of the configuration variables should be consistent with those in the - ``valid_param_vals script``. In addition, various example configuration files can be - found in the ``regional_workflow/tests/baseline_configs`` directory. + ``valid_param_vals script``. In addition, various example configuration files can be found in the ``regional_workflow/tests/baseline_configs`` directory. .. _VXConfig: @@ -715,8 +788,8 @@ These tasks are independent, so users may set some values to "TRUE" and others t .. _SetUpPythonEnv: -Set up the Python and other Environment Parameters --------------------------------------------------- +Set Up the Python and Other Environment Parameters +---------------------------------------------------- The workflow requires Python 3 with the packages 'PyYAML', 'Jinja2', and 'f90nml' available. This Python environment has already been set up on Level 1 platforms, and it can be activated in the following way (from ``/ufs-srweather-app/regional_workflow/ush``): .. code-block:: console @@ -733,6 +806,143 @@ This command will activate the ``regional_workflow`` conda environment. The user source ~/.bashrc conda activate regional_workflow +.. _MacConfig: + +Configuring an Experiment on MacOS +------------------------------------------------------------ + +In principle, the configuration process for MacOS systems is the same as for other systems. However, the details of the configuration process on MacOS require a few extra steps. + +.. note:: + Examples in this subsection presume that the user is running Terminal.app with a bash shell environment. If this is not the case, users will need to adjust the commands to fit their command line application and shell environment. + +.. _MacMorePackages: + +Install Additional Packages +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Check the version of bash, and upgrade it if it is lower than 4. Additionally, install the ``coreutils`` package: + +.. code-block:: console + + bash --version + brew upgrade bash + brew install coreutils + +.. _MacVEnv: + +Create a Python Virtual Environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Users must create a python virtual environment for running the SRW on MacOS. This involves setting python3 as default, adding required python modules, and sourcing the ``regional_workflow``. + +.. code-block:: console + + python3 -m pip --version + python3 -m pip install --upgrade pip + python3 -m ensurepip --default-pip + python3 -m venv $HOME/venv/regional_workflow + source $HOME/venv/regional_workflow/bin/activate + python3 -m pip install jinja2 + python3 -m pip install pyyaml + python3 -m pip install f90nml + python3 -m pip install ruby OR: brew install ruby + +The virtual environment can be deactivated by running the ``deactivate`` command. The virtual environment built here will be reactivated in :numref:`Step %s ` and needs to be used to generate the workflow and run the experiment. + +Install Rocoto +^^^^^^^^^^^^^^^^^^ + +.. note:: + Users may `install Rocoto `__ if they want to make use of a workflow manager to run their experiments. However, this option has not been tested yet on MacOS and is not supported for this release. + + +Configure the SRW App +^^^^^^^^^^^^^^^^^^^^^^^^ + +Users will need to configure their experiment just like on any other system. From the ``$SRW/regional_workflow/ush`` directory, users can copy the settings from ``config.community.sh`` into a ``config.sh`` file (see :numref:`Section %s `) above. In the ``config.sh`` file, users should set ``MACHINE="macos"`` and modify additional variables as needed. For example: + +.. code-block:: console + + MACHINE="macos" + ACCOUNT="user" + EXPT_SUBDIR="" + COMPILER="gnu" + VERBOSE="TRUE" + RUN_ENVIR="community" + PREEXISTING_DIR_METHOD="rename" + + PREDEF_GRID_NAME="RRFS_CONUS_25km" + QUILTING="TRUE" + +Due to the limited number of processors on Mac OS systems, users must configure the domain decomposition defaults (usually, there are only 8 CPUs in M1-family chips and 4 CPUs for x86_64). + +For :ref:`Option 1 `, add the following information to ``config.sh``: + +.. code-block:: console + + LAYOUT_X="${LAYOUT_X:-3}" + LAYOUT_Y="${LAYOUT_Y:-2}" + WRTCMP_write_groups="1" + WRTCMP_write_tasks_per_group="2" + +For :ref:`Option 2 `, add the following information to ``config.sh``: + +.. code-block:: console + + LAYOUT_X="${LAYOUT_X:-3}" + LAYOUT_Y="${LAYOUT_Y:-1}" + WRTCMP_write_groups="1" + WRTCMP_write_tasks_per_group="1" + +Configure the Machine File +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Configure the machine file based on the number of CPUs in the system (8 or 4). Specify the following variables in ``$SRW/regional_workflow/ush/machine/macos.sh``: + +For Option 1 (8 CPUs): + +.. code-block:: console + + # Commands to run at the start of each workflow task. + PRE_TASK_CMDS='{ ulimit -a; }' + + # Architecture information + WORKFLOW_MANAGER="none" + NCORES_PER_NODE=${NCORES_PER_NODE:-8} (Option 2: when 4 CPUs, set to 4) + SCHED=${SCHED:-"none"} + + # UFS SRW App specific paths + FIXgsm="path/to/FIXgsm/files" + FIXaer="path/to/FIXaer/files" + FIXlut="path/to/FIXlut/files" + TOPO_DIR="path/to/FIXgsm/files" # (path to location of static input files + used by the ``make_orog`` task) + SFC_CLIMO_INPUT_DIR="path/to/FIXgsm/files" # (path to location of static surface climatology + input fields used by ``sfc_climo_gen``) + + # Run commands for executables + RUN_CMD_SERIAL="time" + RUN_CMD_UTILS="mpirun -np 4" + RUN_CMD_FCST='mpirun -np ${PE_MEMBER01}' + RUN_CMD_POST="mpirun -np 4" + PRE_TASK_CMDS='{ulimit -a;}' + +The same settings can be used for Option 2, except that ``NCORES_PER_NODE=${NCORES_PER_NODE:-8}`` should be set to 4 instead of 8. + +.. _MacActivateWFenv: + +Activate the Workflow Environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``regional_workflow`` environment can be activated on MacOS as it is for any other system: + +.. code-block:: console + + cd $SRW/regional_workflow/ush + module load wflow_macos + +This should activate the ``regional_workflow`` environment created in :numref:`Step %s `. From here, the user may continue to the :ref:`next step ` and generate the regional workflow. + .. _GenerateWorkflow: @@ -941,7 +1151,7 @@ If the login shell is csh/tcsh, it can be set using: .. code-block:: console - setenv EXPTDIR /path-to-experiment/directory + setenv EXPTDIR // Launch the Rocoto Workflow Using a Script @@ -1104,6 +1314,7 @@ After finishing the experiment, open the crontab using ``crontab -e`` and delete On Orion, *cron* is only available on the orion-login-1 node, so users will need to work on that node when running *cron* jobs on Orion. + The workflow run is complete when all tasks have "SUCCEEDED", and the rocotostat command outputs a table similar to the one :ref:`above `. .. _PlotOutput: diff --git a/docs/UsersGuide/source/Quickstart.rst b/docs/UsersGuide/source/Quickstart.rst index 64685d8a02..3e715a998f 100644 --- a/docs/UsersGuide/source/Quickstart.rst +++ b/docs/UsersGuide/source/Quickstart.rst @@ -334,6 +334,12 @@ Check the batch script output file in your experiment directory for a “SUCCESS | | | | forecast hour) | +------------+------------------------+----------------+----------------------------+ +Users can access log files for specific tasks in the ``$EXPTDIR/log`` directory. To see how the experiment is progressing, users can also check the end of the ``log.launch_FV3LAM_wflow`` file from the command line: + +.. code-block:: console + + tail -n 40 log.launch_FV3LAM_wflow + .. hint:: If any of the scripts return an error that "Primary job terminated normally, but one process returned a non-zero exit code," there may not be enough space on one node to run the process. On an HPC system, the user will need to allocate a(nother) compute node. The process for doing so is system-dependent, and users should check the documentation available for their HPC system. Instructions for allocating a compute node on NOAA Cloud systems can be viewed in the :numref:`Step %s ` as an example. diff --git a/docs/UsersGuide/source/WE2Etests.rst b/docs/UsersGuide/source/WE2Etests.rst index cb4b0a8673..4162395f96 100644 --- a/docs/UsersGuide/source/WE2Etests.rst +++ b/docs/UsersGuide/source/WE2Etests.rst @@ -327,6 +327,7 @@ above, such as ``wflow_features``: Adding a New WE2E Test Category ----------------------------------- + To create a new test category called, e.g., ``new_category``: #. In the directory ``ufs-srweather-app/regional_workflow/tests/WE2E/test_configs``, create a new directory named ``new_category``. From ef24e7a0f47785394970a4c5ac26609f8a883e58 Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Fri, 27 May 2022 00:56:45 -0600 Subject: [PATCH 03/24] Update documentation for CSV file containing WE2E test info (#278) * Edits to documentation to match latest in code. * Edits to documentation to match latest in code. * Minor changes to documentation. --- docs/UsersGuide/source/WE2Etests.rst | 107 ++++++++++++++++++++------- 1 file changed, 79 insertions(+), 28 deletions(-) diff --git a/docs/UsersGuide/source/WE2Etests.rst b/docs/UsersGuide/source/WE2Etests.rst index 4162395f96..1c50a1f0e2 100644 --- a/docs/UsersGuide/source/WE2Etests.rst +++ b/docs/UsersGuide/source/WE2Etests.rst @@ -162,6 +162,7 @@ The full usage statement for ``run_WE2E_tests.sh`` is as follows: [use_cron_to_relaunch="..."] \ [cron_relaunch_intvl_mnts="..."] \ [verbose="..."] \ + [generate_csv_file="..."] \ [machine_file="..."] \ [stmp="..."] \ [ptmp="..."] \ @@ -183,7 +184,8 @@ in the directory ``ufs-srweather-app/regional_workflow/tests/WE2E``. The WE2E Test Information File ================================ In addition to creating the WE2E tests' experiment directories and optionally creating -cron jobs to launch their workflows, the ``run_WE2E_tests.sh`` script generates (if necessary) +cron jobs to launch their workflows, the ``run_WE2E_tests.sh`` script generates (if +necessary and if not explicitly disabled by the ``generate_csv_file="FALSE"`` flag to this script) a CSV (Comma-Separated Value) file named ``WE2E_test_info.csv`` that contains information on the full set of WE2E tests. This file serves as a single location where relevant information about the WE2E tests can be found. It can be imported into Google Sheets @@ -191,27 +193,61 @@ using the "|" (pipe symbol) character as the custom field separator. The rows of file/sheet represent the full set of available tests (not just the ones to be run), while the columns contain the following information (column titles are included in the CSV file): -| Column 1: -| The primary test name and (in parentheses) the category subdirectory it is - located in. +| **Column 1** +| The primary test name followed by the category subdirectory it is + located in (the latter in parentheses). -| Column 2: -| Any alternate names for the test (if any) followed by their category subdirectories +| **Column 2** +| Any alternate names for the test followed by their category subdirectories (in parentheses). -| Column 3: +| **Column 3** | The test description. -| Column 4: -| The number of times the forecast model will be run by the test. This gives an idea - of how expensive the test is. It is calculated using quantities such as the number - of cycle dates (i.e. forecast model start dates) and the number of of ensemble members - (if running ensemble forecasts). The are in turn obtained directly or indirectly - from the quantities in Columns 5, 6, .... +| **Column 4** +| The relative cost of running the dynamics in the test. This gives an + idea of how expensive the test is relative to a reference test that runs + a single 6-hour forecast on the ``RRFS_CONUS_25km`` predefined grid using + its default time step. To calculate the relative cost, the absolute cost + ``abs_cost`` is first calculated as follows: -| Columns 5,6,...: -| The values of various experiment variables (if defined) in each test's configuration - file. Currently, the following experiment variables are included: +.. code-block:: + + abs_cost = nx*ny*num_time_steps*num_fcsts + +| Here, ``nx`` and ``ny`` are the number of grid points in the horizontal + (``x`` and ``y``) directions, ``num_time_steps`` is the number of time + steps in one forecast, and ``num_fcsts`` is the number of forecasts the + test runs (see Column 5 below). [Note that this cost calculation does + not (yet) differentiate between different physics suites.] The relative + cost ``rel_cost`` is then calculated using + +.. code-block:: + + rel_cost = abs_cost/abs_cost_ref + +| where ``abs_cost_ref`` is the absolute cost of running the reference forecast + described above, i.e. a single (``num_fcsts = 1``) 6-hour forecast + (``FCST_LEN_HRS = 6``) on the ``RRFS_CONUS_25km grid`` (which currently has + ``nx = 219``, ``ny = 131``, and ``DT_ATMOS = 40 sec`` (so that ``num_time_steps + = FCST_LEN_HRS*3600/DT_ATMOS = 6*3600/40 = 540``), i.e. + +.. code-block:: + + abs_cost_ref = 219*131*540*1 = 15,492,060 + +| **Column 5** +| The number of times the forecast model will be run by the test. This + is calculated using quantities such as the number of cycle dates (i.e. + forecast model start dates) and the number of of ensemble members (which + is greater than 1 if running ensemble forecasts and 1 otherwise). The + latter are in turn obtained directly or indirectly from the quantities + in Columns 6, 7, .... + +| **Columns 6, 7, ...** +| The values of various experiment variables (if defined) in each test's + configuration file. Currently, the following experiment variables are + included: | ``PREDEF_GRID_NAME`` | ``CCPP_PHYS_SUITE`` @@ -222,18 +258,27 @@ while the columns contain the following information (column titles are included | ``CYCL_HRS`` | ``INCR_CYCL_FREQ`` | ``FCST_LEN_HRS`` + | ``DT_ATMOS`` | ``LBC_SPEC_INTVL_HRS`` | ``NUM_ENS_MEMBERS`` -Additional fields (columns) will likely be added to the CSV file in the near future. +Additional fields (columns) may be added to the CSV file in the future. Note that the CSV file is not part of the ``regional_workflow`` repo (i.e. it is not tracked by the repo). The ``run_WE2E_tests.sh`` script will generate a CSV -file if (1) the CSV file doesn't already exist, or (2) the CSV file does exist -but changes have been made to one or more of the category subdirectories (e.g., -test configuration files modified, added, or deleted) since the creation of the -CSV file. Thus, ``run_WE2E_tests.sh`` will always create a CSV file the first -time it is run in a fresh git clone of the SRW App. +file if the ``generate_csv_file`` flag to this script has not explicitly been +set to ``"FALSE"`` and if either one of the following is true: + +#. The CSV file doesn't already exist. +#. The CSV file does exist, but changes have been made to one or more of the + category subdirectories (e.g., test configuration files modified, added, + or deleted) since the creation of the CSV file. + +Thus, unless the ``generate_csv_file`` flag is set to ``"FALSE"``, the +``run_WE2E_tests.sh`` will create a CSV file the first time it is run in a +fresh git clone of the SRW App. (The ``generate_csv_file`` flag is provided +because the CSV file generation can be slow, so users may wish to skip this +step since it is not a necessary part of running the tests.) Checking Test Status @@ -373,9 +418,14 @@ In this situation, the primary name for the test is ``grid_RRFS_CONUScompact_25k Note the following: -* A primary test can have more than one alternate test name (by having more than one symlink point to the test's configuration file). -* The symlinks representing the alternate test names can be in the same or a different category directory. -* The --relative flag makes the symlink relative (i.e., within/below the ``regional_workflow`` directory structure) so that it stays valid when copied to other locations. However, the ``--relative`` flag may be different and/or not exist on every platform. +* A primary test can have more than one alternate test name (by having + more than one symlink point to the test's configuration file). +* The symlinks representing the alternate test names can be in the same + or a different category directory. +* The ``--relative`` flag makes the symlink relative (i.e., within/below + the ``regional_workflow`` directory structure) so that it stays valid + when copied to other locations. (Note however that this flag is + platform-dependent and may not exist on some platform.) * To determine whether a test has one or more alternate names, a user can view the CSV file ``WE2E_test_info.csv`` that ``run_WE2E_tests.sh`` generates. Recall from :numref:`Section %s ` that column 1 of this CSV @@ -383,8 +433,9 @@ Note the following: any alternate names (and their categories). * With this primary/alternate test naming convention, a user can list either the primary test name or one of the alternate test names in the experiments list file - (e.g. ``my_tests.txt``) that ``run_WE2E_tests.sh`` reads in. If both primary and - one or more alternate test names are listed, then ``run_WE2E_tests.sh`` will exit - with a warning message without running any tests. + (e.g. ``my_tests.txt``) read in by ``run_WE2E_tests.sh``. If more than one name + is listed for the same test (e.g. the primary name and and an alternate name, + two alternate names, etc), ``run_WE2E_tests.sh`` will exit with a warning message + without running any tests. From 9d159caedbd0d20c948354f44f1421aec865095a Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 31 May 2022 10:11:29 -0400 Subject: [PATCH 04/24] Port SRW App to WCOSS2 (#270) * port to wcoss2 * update scripts * ensure platform name from variance * update scripts * remove wcoss2 from lmod-setup --- Externals.cfg | 4 +-- devbuild.sh | 3 +- etc/lmod-setup.sh | 1 - modulefiles/build_wcoss2_intel | 51 ++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 modulefiles/build_wcoss2_intel diff --git a/Externals.cfg b/Externals.cfg index 4efc0448ee..19f1062062 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -12,7 +12,7 @@ protocol = git repo_url = https://github.com/ufs-community/UFS_UTILS # Specify either a branch name or a hash but not both. #branch = develop -hash = 31271f7 +hash = b6efa86 local_path = src/UFS_UTILS required = True @@ -30,7 +30,7 @@ protocol = git repo_url = https://github.com/NOAA-EMC/UPP # Specify either a branch name or a hash but not both. #branch = develop -hash = 394917e +hash = fbd41a5 local_path = src/UPP required = True diff --git a/devbuild.sh b/devbuild.sh index 35a78e3115..08d35c944f 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -10,7 +10,7 @@ OPTIONS show this help guide -p, --platform=PLATFORM name of machine you are building on - (e.g. cheyenne | hera | jet | orion | wcoss_dell_p3) + (e.g. cheyenne | hera | jet | orion | wcoss_dell_p3 | wcoss2) -c, --compiler=COMPILER compiler to use; default depends on platform (e.g. intel | gnu | cray | gccgfortran) @@ -155,6 +155,7 @@ if [ -z "${COMPILER}" ] ; then jet|hera|gaea) COMPILER=intel ;; orion) COMPILER=intel ;; wcoss_dell_p3) COMPILER=intel ;; + wcoss2) COMPILER=intel ;; cheyenne) COMPILER=intel ;; macos,singularity) COMPILER=gnu ;; odin) COMPILER=intel ;; diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index 1a3a813f40..e7a1f95b46 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -38,4 +38,3 @@ elif [ "$L_MACHINE" = odin ]; then else module purge fi - diff --git a/modulefiles/build_wcoss2_intel b/modulefiles/build_wcoss2_intel new file mode 100644 index 0000000000..f7adefb4f7 --- /dev/null +++ b/modulefiles/build_wcoss2_intel @@ -0,0 +1,51 @@ +#%Module + +proc ModulesHelp { } { + puts stderr "This module loads libraries for building SRW on" + puts stderr "the NOAA WCOSS2 machine using Intel-19.1.3.304" +} + +module-whatis "Loads libraries needed for building SRW on WCOSS2 (Cactus/Dogwood)" + +module load envvar/1.0 + +module load PrgEnv-intel/8.1.0 +module load intel/19.1.3.304 +module load craype/2.7.13 +module load cray-mpich/8.1.7 + +module load cmake/3.20.2 + +setenv HPC_OPT /apps/ops/para/libs +module use /apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304 +module use /apps/ops/para/libs/modulefiles/mpi/intel/19.1.3.304/cray-mpich/8.1.7 + +module load jasper/2.0.25 +module load zlib/1.2.11 +module load libpng/1.6.37 +module load hdf5/1.10.6 +module load netcdf/4.7.4 +module load pio/2.5.2 +module load esmf/8.3.0b09 +module load fms/2022.01 +module load bacio/2.4.1 +module load crtm/2.3.0 +module load g2/3.4.5 +module load g2tmpl/1.10.0 +module load ip/3.3.3 +module load sp/2.3.3 +module load w3nco/2.4.1 + +module load libjpeg/9c +module load cray-pals/1.1.3 + +module load w3emc/2.9.2 +module load nemsio/2.5.2 +module load sigio/2.3.2 +module load sfcio/1.4.1 +module load wrf_io/1.2.0 + +setenv CMAKE_C_COMPILER cc +setenv CMAKE_CXX_COMPILER CC +setenv CMAKE_Fortran_COMPILER ftn +setenv CMAKE_Platform wcoss2 From 564e15af3657ef7bc60ec29f3ff4c17459659127 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Tue, 31 May 2022 14:53:15 -0400 Subject: [PATCH 05/24] Fix --ccpp option in devbuild.sh (#280) * fix ccpp issue * update script --- devbuild.sh | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/devbuild.sh b/devbuild.sh index 08d35c944f..df14896717 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -14,11 +14,11 @@ OPTIONS -c, --compiler=COMPILER compiler to use; default depends on platform (e.g. intel | gnu | cray | gccgfortran) - --app=APPLICATION + -a, --app=APPLICATION weather model application to build (e.g. ATM | ATMW | S2S | S2SW) --ccpp="CCPP_SUITE1,CCPP_SUITE2..." - CCCP suites to include in build; delimited with ',' + CCPP suites (CCPP_SUITES) to include in build; delimited with ',' --enable-options="OPTION1,OPTION2,..." enable ufs-weather-model options; delimited with ',' (e.g. 32BIT | INLINE_POST | UFS_GOCART | MOM6 | CICE6 | WW3 | CMEPS) @@ -52,13 +52,13 @@ settings () { cat << EOF_SETTINGS Settings: - SRC_DIR=${SRC_DIR} + SRW_DIR=${SRW_DIR} BUILD_DIR=${BUILD_DIR} INSTALL_DIR=${INSTALL_DIR} PLATFORM=${PLATFORM} COMPILER=${COMPILER} APP=${APPLICATION} - CCPP=${CCPP} + CCPP=${CCPP_SUITES} ENABLE_OPTIONS=${ENABLE_OPTIONS} DISABLE_OPTIONS=${DISABLE_OPTIONS} CLEAN=${CLEAN} @@ -79,13 +79,13 @@ usage_error () { # default settings LCL_PID=$$ -SRC_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) -MACHINE_SETUP=${SRC_DIR}/src/UFS_UTILS/sorc/machine-setup.sh -BUILD_DIR=${SRC_DIR}/build -INSTALL_DIR=${SRC_DIR} +SRW_DIR=$(cd "$(dirname "$(readlink -f -n "${BASH_SOURCE[0]}" )" )" && pwd -P) +MACHINE_SETUP=${SRW_DIR}/src/UFS_UTILS/sorc/machine-setup.sh +BUILD_DIR="${SRW_DIR}/build" +INSTALL_DIR=${SRW_DIR} COMPILER="" APPLICATION="" -CCPP="" +CCPP_SUITES="" ENABLE_OPTIONS="" DISABLE_OPTIONS="" BUILD_TYPE="RELEASE" @@ -108,9 +108,9 @@ while :; do --platform|--platform=|-p|-p=) usage_error "$1 requires argument." ;; --compiler=?*|-c=?*) COMPILER=${1#*=} ;; --compiler|--compiler=|-c|-c=) usage_error "$1 requires argument." ;; - --app=?*) APPLICATION=${1#*=} ;; - --app|--app=) usage_error "$1 requires argument." ;; - --ccpp=?*) CCPP=${1#*=} ;; + --app=?*|-a=?*) APPLICATION=${1#*=} ;; + --app|--app=|-a|-a=) usage_error "$1 requires argument." ;; + --ccpp=?*) CCPP_SUITES=${1#*=} ;; --ccpp|--ccpp=) usage_error "$1 requires argument." ;; --enable-options=?*) ENABLE_OPTIONS=${1#*=} ;; --enable-options|--enable-options=) usage_error "$1 requires argument." ;; @@ -136,6 +136,11 @@ while :; do shift done +# Ensure uppercase / lowercase ============================================ +APPLICATION="${APPLICATION^^}" +PLATFORM="${PLATFORM,,}" +COMPILER="${COMPILER,,}" + # check if PLATFORM is set if [ -z $PLATFORM ] ; then printf "\nERROR: Please set PLATFORM.\n\n" @@ -175,7 +180,7 @@ fi # set MODULE_FILE for this platform/compiler combination MODULE_FILE="build_${PLATFORM}_${COMPILER}" -if [ ! -f "${SRC_DIR}/modulefiles/${MODULE_FILE}" ]; then +if [ ! -f "${SRW_DIR}/modulefiles/${MODULE_FILE}" ]; then printf "ERROR: module file does not exist for platform/compiler\n" >&2 printf " MODULE_FILE=${MODULE_FILE}\n" >&2 printf " PLATFORM=${PLATFORM}\n" >&2 @@ -227,8 +232,8 @@ CMAKE_SETTINGS="${CMAKE_SETTINGS} -DCMAKE_BUILD_TYPE=${BUILD_TYPE}" if [ ! -z "${APPLICATION}" ]; then CMAKE_SETTINGS="${CMAKE_SETTINGS} -DAPP=${APPLICATION}" fi -if [ ! -z "${CCPP}" ]; then - CMAKE_SETTINGS="${CMAKE_SETTINGS} -DCCPP=${CCPP}" +if [ ! -z "${CCPP_SUITES}" ]; then + CMAKE_SETTINGS="${CMAKE_SETTINGS} -DCCPP_SUITES=${CCPP_SUITES}" fi if [ ! -z "${ENABLE_OPTIONS}" ]; then CMAKE_SETTINGS="${CMAKE_SETTINGS} -DENABLE_OPTIONS=${ENABLE_OPTIONS}" @@ -244,17 +249,17 @@ if [ "${VERBOSE}" = true ]; then fi # Before we go on load modules, we first need to activate Lmod for some systems -source ${SRC_DIR}/etc/lmod-setup.sh +source ${SRW_DIR}/etc/lmod-setup.sh # source the module file for this platform/compiler combination, then build the code printf "... Load MODULE_FILE and create BUILD directory ...\n" -module use ${SRC_DIR}/modulefiles +module use ${SRW_DIR}/modulefiles module load ${MODULE_FILE} module list mkdir -p ${BUILD_DIR} cd ${BUILD_DIR} printf "... Generate CMAKE configuration ...\n" -cmake ${SRC_DIR} ${CMAKE_SETTINGS} 2>&1 | tee log.cmake +cmake ${SRW_DIR} ${CMAKE_SETTINGS} 2>&1 | tee log.cmake printf "... Compile executables ...\n" make ${MAKE_SETTINGS} 2>&1 | tee log.make From c118a5fee0a94f0f3ffc6a1e8a9a8dc8561f7dc8 Mon Sep 17 00:00:00 2001 From: danielabdi-noaa <52012304+danielabdi-noaa@users.noreply.github.com> Date: Tue, 31 May 2022 13:08:02 -0600 Subject: [PATCH 06/24] Print usage if machine name is not passed to `lmod-setup.sh/csh`. (#262) * Print usage() message if machine name is not passed. * Bug fix with macos modulefiles. --- devbuild.sh | 4 ++-- docs/INSTALL | 2 +- etc/lmod-setup.csh | 9 ++++++++- etc/lmod-setup.sh | 8 ++++++++ modulefiles/build_macos_gnu | 14 ++++++++++++-- modulefiles/wflow_macos | 5 ++++- test/build.sh | 2 +- 7 files changed, 36 insertions(+), 8 deletions(-) diff --git a/devbuild.sh b/devbuild.sh index df14896717..aeb4358d51 100755 --- a/devbuild.sh +++ b/devbuild.sh @@ -163,7 +163,7 @@ if [ -z "${COMPILER}" ] ; then wcoss2) COMPILER=intel ;; cheyenne) COMPILER=intel ;; macos,singularity) COMPILER=gnu ;; - odin) COMPILER=intel ;; + odin,noaacloud) COMPILER=intel ;; *) COMPILER=intel printf "WARNING: Setting default COMPILER=intel for new platform ${PLATFORM}\n" >&2; @@ -249,7 +249,7 @@ if [ "${VERBOSE}" = true ]; then fi # Before we go on load modules, we first need to activate Lmod for some systems -source ${SRW_DIR}/etc/lmod-setup.sh +source ${SRW_DIR}/etc/lmod-setup.sh $MACHINE # source the module file for this platform/compiler combination, then build the code printf "... Load MODULE_FILE and create BUILD directory ...\n" diff --git a/docs/INSTALL b/docs/INSTALL index 4b659bb1d1..e53044f6ad 100644 --- a/docs/INSTALL +++ b/docs/INSTALL @@ -42,7 +42,7 @@ source etc/lmod-setup.sh PLATFORM # From here on, we can assume Lmod is loaded and ready to go. Then we load the specific # module for a given PLATFORM and COMPILER as follows -module use modulefiles +module use $PWD/modulefiles #full path to modulefiles directory module load build_[PLATFORM]_[COMPILER] # Supported CMake flags: diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh index f63095a28f..d891bc07ec 100644 --- a/etc/lmod-setup.csh +++ b/etc/lmod-setup.csh @@ -1,7 +1,14 @@ #!/bin/csh if ( $# == 0 ) then - set L_MACHINE=${MACHINE} + cat << EOF_USAGE +Usage: source etc/lmod-setup.csh PLATFORM + +OPTIONS: + PLATFORM - name of machine you are building on + (e.g. cheyenne | hera | jet | orion | wcoss_dell_p3) +EOF_USAGE + exit 1 else set L_MACHINE=$1 endif diff --git a/etc/lmod-setup.sh b/etc/lmod-setup.sh index e7a1f95b46..c2d4cc92a0 100644 --- a/etc/lmod-setup.sh +++ b/etc/lmod-setup.sh @@ -2,6 +2,14 @@ if [ $# = 0 ]; then L_MACHINE=${MACHINE} + cat << EOF_USAGE +Usage: source etc/lmod-setup.sh PLATFORM + +OPTIONS: + PLATFORM - name of machine you are building on + (e.g. cheyenne | hera | jet | orion | wcoss_dell_p3) +EOF_USAGE + exit 1 else L_MACHINE=$1 fi diff --git a/modulefiles/build_macos_gnu b/modulefiles/build_macos_gnu index 7947b2a343..4277bf45d7 100644 --- a/modulefiles/build_macos_gnu +++ b/modulefiles/build_macos_gnu @@ -8,7 +8,7 @@ proc ModulesHelp { } { module-whatis "Loads libraries needed for building SRW on Hera" if { [module-info mode load] } { - system "ulimit -S -s unlimited;" + puts "ulimit -S -s unlimited;" } # This path should point to your HPCstack installation directory @@ -79,5 +79,15 @@ setenv CMAKE_Fortran_COMPILER $env(MPI_FC) setenv CMAKE_Platform macos.gnu setenv CMAKE_Fortran_COMPILER_ID "GNU" -setenv LDFLAGS "-L$env(MPI_ROOT)/lib" setenv FFLAGS "-DNO_QUAD_PRECISION -fallow-argument-mismatch " + +#leave setting LDFLAGS to user because MPI_ROOT does not have +#valid value before this modulefile is fully loaded. +set shell [module-info shelltype] +if {$shell == "sh"} { + puts stderr {Please execute this command: + > export LDFLAGS=-L$MPI_ROOT/lib} +} else { + puts stderr {Please execute this command: + > setenv LDFLAGS -L$MPI_ROOT/lib} +} diff --git a/modulefiles/wflow_macos b/modulefiles/wflow_macos index 7481b36911..07b5980208 100644 --- a/modulefiles/wflow_macos +++ b/modulefiles/wflow_macos @@ -14,7 +14,10 @@ setenv CMAKE_Platform macos # setenv VENV "/Users/username/venv/regional_workflow" if { [module-info mode load] } { - system "source $env(VENV)/bin/activate;" + puts "source $env(VENV)/bin/activate;" +} +if { [module-info mode remove] } { + puts "deactivate;" } # Uncomment if Rocoto workflow manager is used diff --git a/test/build.sh b/test/build.sh index 83c30910e7..cb2119066d 100755 --- a/test/build.sh +++ b/test/build.sh @@ -21,7 +21,7 @@ function usage() { exit 1 } -machines=( hera jet cheyenne orion wcoss_cray wcoss_dell_p3 gaea odin singularity ) +machines=( hera jet cheyenne orion wcoss2 wcoss_dell_p3 gaea odin singularity macos noaacloud ) [[ $# -eq 2 ]] && usage From 1fe6fc85e486dbe89ad95274c02c629e386874a3 Mon Sep 17 00:00:00 2001 From: "Chan-Hoo.Jeon-NOAA" <60152248+chan-hoo@users.noreply.github.com> Date: Wed, 1 Jun 2022 14:55:26 -0400 Subject: [PATCH 07/24] Back to original hashes (#287) --- Externals.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 19f1062062..4efc0448ee 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -12,7 +12,7 @@ protocol = git repo_url = https://github.com/ufs-community/UFS_UTILS # Specify either a branch name or a hash but not both. #branch = develop -hash = b6efa86 +hash = 31271f7 local_path = src/UFS_UTILS required = True @@ -30,7 +30,7 @@ protocol = git repo_url = https://github.com/NOAA-EMC/UPP # Specify either a branch name or a hash but not both. #branch = develop -hash = fbd41a5 +hash = 394917e local_path = src/UPP required = True From ae7a11ccb6e868bad0231c656c923a2bd22a7a31 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Thu, 2 Jun 2022 09:58:53 -0400 Subject: [PATCH 08/24] Updated the Introduction, build for MacOS (#281) * Updated the Introduction, build for MacOS * Update Introduction.rst Some comments removed * Update Introduction.rst Updated data storage requirements for the SRW --- docs/UsersGuide/source/BuildRunSRW.rst | 19 ++++----- docs/UsersGuide/source/Introduction.rst | 52 ++++++++++++------------- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/docs/UsersGuide/source/BuildRunSRW.rst b/docs/UsersGuide/source/BuildRunSRW.rst index 658ff72a34..94e7deda10 100644 --- a/docs/UsersGuide/source/BuildRunSRW.rst +++ b/docs/UsersGuide/source/BuildRunSRW.rst @@ -238,6 +238,7 @@ Set Up the Workflow Environment .. attention:: If users successfully built the executables in :numref:`Step %s `, they should skip to step :numref:`Step %s `. + For the CMake steps on MacOS systems, follow the approach in :numref:`Step %s `. If the ``devbuild.sh`` approach failed, users need to set up their environment to run a workflow on their specific platform. First, users should make sure ``Lmod`` is the app used for loading modulefiles. This is the case on most Level 1 systems; however, on systems such as Gaea/Odin, the default modulefile loader is from Cray and must be switched to Lmod. For example, on Gaea, assuming a ``bash`` login shell, run: @@ -339,25 +340,20 @@ Then, users must source the Lmod setup file, just as they would on other systems source etc/lmod-setup.sh macos module use module load build_macos_gnu + export LDFLAGS="-L${MPI_ROOT}/lib" In a csh/tcsh shell, users would run ``source etc/lmod-setup.csh macos`` in place of the first line in the code above. .. note:: If you execute ``source etc/lmod-setup.sh`` on systems that don't need it, it will simply do a ``module purge``. -Additionally, for Option 1 systems, set the variable ``ENABLE_QUAD_PRECISION`` to ``OFF`` in line 35 of the ``$SRW/src/ufs-weather-model/FV3/atmos_cubed_sphere/CMakeLists.txt`` file. This change is optional if using Option 2 to build the SRW App. Using a text editor (e.g., vi, vim, emacs): +Additionally, for Option 1 systems, set the variable ``ENABLE_QUAD_PRECISION`` to ``OFF`` in ``$SRW/src/ufs-weather-model/FV3/atmos_cubed_sphere/CMakeLists.txt`` file. This change is optional if using Option 2 to build the SRW App. You could use a streamline editor `sed` to change it: .. code-block:: console - option(ENABLE_QUAD_PRECISION "Enable compiler definition -DENABLE_QUAD_PRECISION" OFF) - -An alternative way to make this change is using a `sed` (streamline editor). From the command line, users can run one of two commands (user's preference): - -.. code-block:: console - - sed -i -e 's/QUAD_PRECISION\" ON)/QUAD_PRECISION\" OFF)/' CMakeLists.txt - sed -i -e 's/bin\/sh/bin\/bash/g' *sh + sed -i .bak 's/QUAD_PRECISION\" ON)/QUAD_PRECISION\" OFF)/' $SRW/src/ufs-weather-model/FV3/atmos_cubed_sphere/CMakeLists.txt +Proceed to building executables using CMake in :numref:`Step %s ` .. _Data: @@ -908,7 +904,7 @@ For Option 1 (8 CPUs): # Architecture information WORKFLOW_MANAGER="none" - NCORES_PER_NODE=${NCORES_PER_NODE:-8} (Option 2: when 4 CPUs, set to 4) + NCORES_PER_NODE=${NCORES_PER_NODE:-8} SCHED=${SCHED:-"none"} # UFS SRW App specific paths @@ -925,9 +921,8 @@ For Option 1 (8 CPUs): RUN_CMD_UTILS="mpirun -np 4" RUN_CMD_FCST='mpirun -np ${PE_MEMBER01}' RUN_CMD_POST="mpirun -np 4" - PRE_TASK_CMDS='{ulimit -a;}' -The same settings can be used for Option 2, except that ``NCORES_PER_NODE=${NCORES_PER_NODE:-8}`` should be set to 4 instead of 8. +Using Option 2 with 4 CPUs requires ``NCORES_PER_NODE=${NCORES_PER_NODE:-4}`` in the above example. .. _MacActivateWFenv: diff --git a/docs/UsersGuide/source/Introduction.rst b/docs/UsersGuide/source/Introduction.rst index 9e1e86b3dc..208162a9e2 100644 --- a/docs/UsersGuide/source/Introduction.rst +++ b/docs/UsersGuide/source/Introduction.rst @@ -14,9 +14,6 @@ The SRW App v1.0.0 citation is as follows and should be used when presenting res UFS Development Team. (2021, March 4). Unified Forecast System (UFS) Short-Range Weather (SRW) Application (Version v1.0.0). Zenodo. https://doi.org/10.5281/zenodo.4534994 -.. - COMMENT: Update version numbers/citation for release! Also update release date for citation! - How to Use This Document ======================== @@ -51,7 +48,7 @@ The instructions in this documentation assume that users have certain background * Familiarity with LINUX/UNIX systems * Command line basics * System configuration knowledge (e.g., compilers, environment variables, paths, etc.) -* Numerical Weather Prediction +* Numerical Weather Prediction (concepts of parameterizations: physical, microphysical, convective) * Meteorology (particularly meteorology at the scales being predicted) .. @@ -73,12 +70,14 @@ The UFS SRW Application has been designed so that any sufficiently up-to-date ma * POSIX-compliant UNIX-style operating system -* >40 GB disk space +* >82 GB disk space + + * 53 GB input data for a standard collection of global database, or "fix" data (topography, climatology, observational database), and boundary conditions for a short 12-h test forecast on CONUS 25km domain. See data download instructions in :numref:`Step %s `. + * 8 GB for :term:`HPC-Stack` full installation + * 3 GB for ufs-srweather-app installation + * 1 GB boundary conditions for a short 12-h test forecast on CONUS 25km domain. See data download instructions in :numref:`Step %s ` + * 17 GB for a 12-h test forecast on CONUS 25km domain, with model output saved hourly, see :numref:`Step %s ` - * 18 GB input data from GFS, RAP, and HRRR for "out-of-the-box" SRW App case described in :numref:`Chapter %s ` - * 6 GB for :term:`HPC-Stack` full installation - * 1 GB for ufs-srweather-app installation - * 11 GB for 48hr forecast on CONUS 25km domain * 4GB memory (CONUS 25km domain) @@ -88,7 +87,7 @@ The UFS SRW Application has been designed so that any sufficiently up-to-date ma * C and C++ compilers compatible with the Fortran compiler - * gcc v9+, ifort v18+, and clang v9+ (macOS, native Apple clang or LLVM clang) have been tested + * gcc v9+, ifort v18+, and clang v9+ (macOS, native Apple clang, LLVM clang, GNU) have been tested * Python v3.6+, including prerequisite packages ``jinja2``, ``pyyaml`` and ``f90nml`` @@ -96,41 +95,37 @@ The UFS SRW Application has been designed so that any sufficiently up-to-date ma * Perl 5 -* git v1.8+ +* git v2.12+ -.. - COMMENT: Should curl/wget/TIFF library also be required? These are listed as prerequisites for building HPC-Stack on generic MacOS/Linux +* curl + +* wget +.. + COMMENT: Should curl/wget/TIFF library also be required? These are listed as prerequisites for building HPC-Stack on generic MacOS/Linux | A: TIFF could be installed with the HPC-stack build; others are needed The following software is also required to run the SRW Application, but the :term:`HPC-Stack` (which contains the software libraries necessary for building and running the SRW App) can be configured to build these requirements: -* CMake v3.15+ +* CMake v3.20+ * MPI (MPICH, OpenMPI, or other implementation) - * Only **MPICH** can be built with HPC-Stack. Other options must be installed separately by the user. - -* Software libraries - - * netCDF (C and Fortran libraries) - * HDF5 - * ESMF 8.2.0 - * Jasper - * libJPG - * libPNG - * zlib + * Only **MPICH** or **OpenMPI** can be built with HPC-Stack. Other implementations must be installed separately by the user. For MacOS systems, some additional software is needed. It is recommended that users install this software using the `Homebrew `__ package manager for MacOS: +* brew install bash + + * OR: brew upgrade bash * brew install gcc@11 * brew install cmake * brew install make * brew install wget +* brew install lmod * brew install coreutils -* brew install pkg-config .. - COMMENT: Is this still accurate? It seems like we should delete the last one. And gcc@11 is basically the same as requiring fortran/C/C++ compilers, no? CMake is listed above. + COMMENT: Is this still accurate? It seems like we should delete the last one. And gcc@11 is basically the same as requiring fortran/C/C++ compilers, no? CMake is listed above. | A: updated bash requirements Optional but recommended prerequisites for all systems: @@ -138,8 +133,9 @@ Optional but recommended prerequisites for all systems: * Bash v4+ * Rocoto Workflow Management System (1.3.1) * Python packages ``scipy``, ``matplotlib``, ``pygrib``, ``cartopy``, and ``pillow`` for graphics -* Lmod +.. + COMMENT: Lmod is listed as required .. _ComponentsOverview: From d69a0af62bb6ecce59527581180ad84d1b89431e Mon Sep 17 00:00:00 2001 From: gsketefian <31046882+gsketefian@users.noreply.github.com> Date: Thu, 2 Jun 2022 17:07:27 -0600 Subject: [PATCH 09/24] Update documentation for python plotting scripts (#289) ## DESCRIPTION OF CHANGES: This PR updates the RST documentation files to accompany changes in PR #[783](https://github.com/ufs-community/regional_workflow/pull/783) in `regional_workflow`. ## DEPENDENCIES: PR #[783](https://github.com/ufs-community/regional_workflow/pull/783) in `regional_workflow`. ## ISSUE (optional): Partially resolves issue #[781](https://github.com/ufs-community/regional_workflow/issues/781) (still need to introduce tests in `regional_workflow` that run the plotting scripts). ## CONTRIBUTORS: @mark-a-potts --- docs/UsersGuide/source/Graphics.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/UsersGuide/source/Graphics.rst b/docs/UsersGuide/source/Graphics.rst index 6243696ce7..828ac1a155 100644 --- a/docs/UsersGuide/source/Graphics.rst +++ b/docs/UsersGuide/source/Graphics.rst @@ -147,6 +147,7 @@ following six command line arguments: #. Forecast hour increment #. The top level of the experiment directory ``EXPTDIR`` containing the post-processed data. The script will look for the data files in the directory ``EXPTDIR/CDATE/postprd``. #. The base directory ``CARTOPY_DIR`` of the cartopy shapefiles. The script will look for the shape files (``*.shp``) in the directory ``CARTOPY_DIR/shapefiles/natural_earth/cultural``. +#. The name ``POST_OUTPUT_DOMAIN_NAME`` of the native grid used in the forecast .. note:: If a forecast starts at 18h, this is considered the 0th forecast hour, so "starting forecast hour" should be 0, not 18. @@ -157,7 +158,7 @@ is as follows: .. code-block:: console - python plot_allvars.py 2019061500 6 48 6 /path-to/expt_dirs/test_CONUS_25km_GFSv16 /path-to/NaturalEarth + python plot_allvars.py 2019061500 6 48 6 /path-to/expt_dirs/test_CONUS_25km_GFSv16 /path-to/NaturalEarth RRFS_CONUS_25km The output files (in ``.png`` format) will be located in the directory ``EXPTDIR/CDATE/postprd``, where in this case ``EXPTDIR`` is ``/path-to/expt_dirs/test_CONUS_25km_GFSv16`` and ``CDATE`` @@ -176,12 +177,13 @@ seven command line arguments: #. The top level of the first experiment directory ``EXPTDIR1`` containing the first set of post-processed data. The script will look for the data files in the directory ``EXPTDIR1/CDATE/postprd``. #. The top level of the first experiment directory ``EXPTDIR2`` containing the second set of post-processed data. The script will look for the data files in the directory ``EXPTDIR2/CDATE/postprd``. #. The base directory ``CARTOPY_DIR`` of the cartopy shapefiles. The script will look for the shape files (``*.shp``) in the directory ``CARTOPY_DIR/shapefiles/natural_earth/cultural``. +#. The name ``POST_OUTPUT_DOMAIN_NAME`` of the native grid used in the forecasts (this must be the same for the two forecasts) An example of plotting differences from two experiments for the same date and predefined domain where one uses the "FV3_GFS_v16" suite definition file (SDF) and one using the "FV3_RRFS_v1beta" SDF is as follows: .. code-block:: console - python plot_allvars_diff.py 2019061518 0 18 6 /path-to/expt_dirs1/test_CONUS_3km_GFSv16 /path-to/expt_dirs2/test_CONUS_3km_RRFSv1beta /path-to/NaturalEarth + python plot_allvars_diff.py 2019061518 0 18 6 /path-to/expt_dirs1/test_CONUS_3km_GFSv16 /path-to/expt_dirs2/test_CONUS_3km_RRFSv1beta /path-to/NaturalEarth RRFS_CONUS_25km In this case, the output ``.png`` files will be located in the directory ``EXPTDIR1/CDATE/postprd``. From 1da4c3ffb1de4c758ae9e80c068878e9f4621d32 Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Tue, 7 Jun 2022 16:28:32 -0500 Subject: [PATCH 10/24] Updating for testing with updated ufs weather model --- Externals.cfg | 7 ++++--- modulefiles/srw_common | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 4efc0448ee..8f5f4a3b51 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -3,7 +3,7 @@ protocol = git repo_url = https://github.com/ufs-community/regional_workflow # Specify either a branch name or a hash but not both. #branch = develop -hash = de82b63 +hash = release/public-v2 local_path = regional_workflow required = True @@ -18,10 +18,11 @@ required = True [ufs-weather-model] protocol = git -repo_url = https://github.com/ufs-community/ufs-weather-model +#repo_url = https://github.com/ufs-community/ufs-weather-model +repo_url = https://github.com/NOAA-EPIC/ufs-weather-model-1 # Specify either a branch name or a hash but not both. #branch = develop -hash = 96dffa1 +hash = release/public-v3 local_path = src/ufs-weather-model required = True diff --git a/modulefiles/srw_common b/modulefiles/srw_common index c8c87f92a8..812877dde4 100644 --- a/modulefiles/srw_common +++ b/modulefiles/srw_common @@ -8,7 +8,7 @@ module load hdf5/1.10.6 module load-any netcdf/4.7.4 netcdf-c/4.7.4 module load-any netcdf/4.7.4 netcdf-fortran/4.5.4 module load-any pio/2.5.2 parallelio/2.5.2 -module load-any esmf/8_2_0 esmf/8.2.0 +module load-any esmf/8.3.0b09 module load fms/2021.03 module load bacio/2.4.1 @@ -22,7 +22,7 @@ module load upp/10.0.10 module load-any gftl-shared/v1.3.3 gftl-shared/1.3.3 module load-any yafyaml/v0.5.1 yafyaml/0.5.1 -module load-any mapl/2.11.0-esmf-8_2_0 mapl/2.11.0-esmf-8.2.0 +module load-any mapl/2.11.0-esmf-8.3.0b09 module load gfsio/1.4.1 module load landsfcutil/2.4.1 From f67a3b41394eca5c3c8f911d6469e11edd740dc8 Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Wed, 8 Jun 2022 06:22:58 -0500 Subject: [PATCH 11/24] added version to wflow for jet --- modulefiles/wflow_jet | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/wflow_jet b/modulefiles/wflow_jet index e340ca4a12..31e94efe91 100644 --- a/modulefiles/wflow_jet +++ b/modulefiles/wflow_jet @@ -10,7 +10,7 @@ module-whatis "Loads libraries needed for running SRW on Jet" module load rocoto module use -a /contrib/miniconda3/modulefiles -module load miniconda3 +module load miniconda3/4.5.12 if { [module-info mode load] } { puts stderr "Please do the following to activate conda: From ae912ec52ade4b26908cdb86bbe689ea2c7aebc3 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Wed, 8 Jun 2022 12:26:23 +0000 Subject: [PATCH 12/24] switched to epic repo for testing --- Externals.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 8f5f4a3b51..fb20ed65d0 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,6 +1,7 @@ [regional_workflow] protocol = git -repo_url = https://github.com/ufs-community/regional_workflow +#repo_url = https://github.com/ufs-community/regional_workflow +repo_url = https://github.com/NOAA-EPIC/regional_workflow # Specify either a branch name or a hash but not both. #branch = develop hash = release/public-v2 From 6bc45c610e3a3cef06da42f7391a12902ba6dc76 Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Wed, 8 Jun 2022 07:29:09 -0600 Subject: [PATCH 13/24] Updates to modules for cheyenne --- modulefiles/build_cheyenne_gnu | 6 ++++-- modulefiles/build_cheyenne_intel | 5 +++-- modulefiles/srw_common | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/modulefiles/build_cheyenne_gnu b/modulefiles/build_cheyenne_gnu index add5ee3b0f..5b2aff3b1b 100644 --- a/modulefiles/build_cheyenne_gnu +++ b/modulefiles/build_cheyenne_gnu @@ -15,14 +15,16 @@ module load ncarcompilers/0.5.0 module load python/3.7.9 module unload netcdf -module use /glade/p/ral/jntp/GMTB/tools/hpc-stack-v1.2.0/modulefiles/stack +module use /glade/work/epicufsrt/GMTB/tools/hpc-stack-v1.2.0_6eb6/modulefiles/stack + module load hpc/1.2.0 module load hpc-gnu/10.1.0 module load hpc-mpt/2.22 module load srw_common module load g2/3.4.3 -module load esmf/8_2_0 +module load esmf/v8.3.0b09 +module load mapl/2.11.0-esmf-8.3.0b09 module load netcdf/4.7.4 module load png/1.6.35 module load pio/2.5.2 diff --git a/modulefiles/build_cheyenne_intel b/modulefiles/build_cheyenne_intel index b83c36a5df..5cc295dbde 100644 --- a/modulefiles/build_cheyenne_intel +++ b/modulefiles/build_cheyenne_intel @@ -15,14 +15,15 @@ module load ncarcompilers/0.5.0 module load python/3.7.9 module unload netcdf -module use /glade/p/ral/jntp/GMTB/tools/hpc-stack-v1.2.0/modulefiles/stack +module use /glade/work/epicufsrt/GMTB/tools/hpc-stack-v1.2.0_6eb6/modulefiles/stack module load hpc/1.2.0 module load hpc-intel/2021.2 module load hpc-mpt/2.22 module load srw_common module load g2/3.4.3 -module load esmf/8_2_0 +module load esmf/8.3.0b09 +module load mapl/2.11.0-esmf-8.3.0b09 module load netcdf/4.7.4 module load png/1.6.35 module load pio/2.5.2 diff --git a/modulefiles/srw_common b/modulefiles/srw_common index 812877dde4..dbf65dacf0 100644 --- a/modulefiles/srw_common +++ b/modulefiles/srw_common @@ -9,7 +9,7 @@ module load-any netcdf/4.7.4 netcdf-c/4.7.4 module load-any netcdf/4.7.4 netcdf-fortran/4.5.4 module load-any pio/2.5.2 parallelio/2.5.2 module load-any esmf/8.3.0b09 -module load fms/2021.03 +module load fms module load bacio/2.4.1 module load crtm/2.3.0 From a27f8ab998fef141829d4e74579e40bc338eb55d Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Wed, 8 Jun 2022 07:41:18 -0600 Subject: [PATCH 14/24] updated module path --- modulefiles/build_cheyenne_gnu | 9 ++++----- modulefiles/build_cheyenne_intel | 6 +++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/modulefiles/build_cheyenne_gnu b/modulefiles/build_cheyenne_gnu index 5b2aff3b1b..9a65f402ec 100644 --- a/modulefiles/build_cheyenne_gnu +++ b/modulefiles/build_cheyenne_gnu @@ -15,15 +15,14 @@ module load ncarcompilers/0.5.0 module load python/3.7.9 module unload netcdf -module use /glade/work/epicufsrt/GMTB/tools/hpc-stack-v1.2.0_6eb6/modulefiles/stack - +module use /glade/work/epicufsrt/GMTB/tools/gnu/11.2.0/hpc-stack-v1.2.0/modulefiles/stack module load hpc/1.2.0 -module load hpc-gnu/10.1.0 -module load hpc-mpt/2.22 +module load hpc-gnu +module load hpc-mpt/2.25 module load srw_common module load g2/3.4.3 -module load esmf/v8.3.0b09 +module load esmf/8.3.0b09 module load mapl/2.11.0-esmf-8.3.0b09 module load netcdf/4.7.4 module load png/1.6.35 diff --git a/modulefiles/build_cheyenne_intel b/modulefiles/build_cheyenne_intel index 5cc295dbde..e2e25ef9f0 100644 --- a/modulefiles/build_cheyenne_intel +++ b/modulefiles/build_cheyenne_intel @@ -15,10 +15,10 @@ module load ncarcompilers/0.5.0 module load python/3.7.9 module unload netcdf -module use /glade/work/epicufsrt/GMTB/tools/hpc-stack-v1.2.0_6eb6/modulefiles/stack +module use /glade/work/epicufsrt/GMTB/tools/intel/2022.1/hpc-stack-v1.2.0_6eb6/modulefiles/stack module load hpc/1.2.0 -module load hpc-intel/2021.2 -module load hpc-mpt/2.22 +module load hpc-intel/2022.1 +module load hpc-mpt/2.25 module load srw_common module load g2/3.4.3 From 8f2d65feecfca6f29471d5c4ad9373815741860b Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Wed, 8 Jun 2022 08:16:07 -0600 Subject: [PATCH 15/24] changed hash to branch for testing --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index fb20ed65d0..cad000e847 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -4,7 +4,7 @@ protocol = git repo_url = https://github.com/NOAA-EPIC/regional_workflow # Specify either a branch name or a hash but not both. #branch = develop -hash = release/public-v2 +branch = release/public-v2 local_path = regional_workflow required = True From e81a249a3dc4734f2caf159f0d959944261d9123 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Wed, 8 Jun 2022 16:33:06 -0400 Subject: [PATCH 16/24] updated to use ufs-community version of WM --- Externals.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index cad000e847..7d68bbe56a 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -19,11 +19,11 @@ required = True [ufs-weather-model] protocol = git -#repo_url = https://github.com/ufs-community/ufs-weather-model -repo_url = https://github.com/NOAA-EPIC/ufs-weather-model-1 +repo_url = https://github.com/ufs-community/ufs-weather-model +#repo_url = https://github.com/NOAA-EPIC/ufs-weather-model-1 # Specify either a branch name or a hash but not both. #branch = develop -hash = release/public-v3 +hash = release-public-v3-SRW local_path = src/ufs-weather-model required = True From f27e30f6b9c362ba2e4e5752edd8cf9f35f51935 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Wed, 8 Jun 2022 16:59:07 -0400 Subject: [PATCH 17/24] Pointing to PR-branch of regional_workflow for testing --- Externals.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 7d68bbe56a..e2ba7dcb9d 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -4,7 +4,7 @@ protocol = git repo_url = https://github.com/NOAA-EPIC/regional_workflow # Specify either a branch name or a hash but not both. #branch = develop -branch = release/public-v2 +branch = feature/hera-fix local_path = regional_workflow required = True From 8ba5a2aab16cafef7467d58e36cfb6a4e2c818a3 Mon Sep 17 00:00:00 2001 From: Mark Potts Date: Thu, 9 Jun 2022 06:35:00 -0600 Subject: [PATCH 18/24] Updating module files for cheyenne for testing --- modulefiles/build_cheyenne_gnu | 6 +++--- modulefiles/srw_common | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/modulefiles/build_cheyenne_gnu b/modulefiles/build_cheyenne_gnu index 9a65f402ec..f8be4be1ea 100644 --- a/modulefiles/build_cheyenne_gnu +++ b/modulefiles/build_cheyenne_gnu @@ -9,8 +9,8 @@ module-whatis "Loads libraries needed for building SRW on Cheyenne" module load cmake/3.22.0 module load ncarenv/1.3 -module load gnu/10.1.0 -module load mpt/2.22 +module load gnu/11.2.0 +module load mpt/2.25 module load ncarcompilers/0.5.0 module load python/3.7.9 module unload netcdf @@ -22,10 +22,10 @@ module load hpc-mpt/2.25 module load srw_common module load g2/3.4.3 +module load w3emc/2.9.2 module load esmf/8.3.0b09 module load mapl/2.11.0-esmf-8.3.0b09 module load netcdf/4.7.4 -module load png/1.6.35 module load pio/2.5.2 setenv CMAKE_C_COMPILER mpicc diff --git a/modulefiles/srw_common b/modulefiles/srw_common index dbf65dacf0..570a492468 100644 --- a/modulefiles/srw_common +++ b/modulefiles/srw_common @@ -2,7 +2,8 @@ module load jasper/2.0.25 module load zlib/1.2.11 -module load-any png/1.6.35 libpng/1.6.35 +module try-load png/1.6.35 +module try-load libpng/1.6.35 module load hdf5/1.10.6 module load-any netcdf/4.7.4 netcdf-c/4.7.4 @@ -30,5 +31,5 @@ module load nemsio/2.5.2 module load nemsiogfs/2.5.3 module load sfcio/1.4.1 module load sigio/2.3.2 -module load w3emc/2.7.3 +module try-load w3emc/2.7.3 module load wgrib2/2.0.8 From 4c37fe0da9d9fa1d3200902404d50e674ed23649 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Thu, 9 Jun 2022 12:45:00 -0400 Subject: [PATCH 19/24] Added png explicitly under gaea. --- modulefiles/build_gaea_intel | 1 + 1 file changed, 1 insertion(+) diff --git a/modulefiles/build_gaea_intel b/modulefiles/build_gaea_intel index 15ed495a90..cce0e07468 100644 --- a/modulefiles/build_gaea_intel +++ b/modulefiles/build_gaea_intel @@ -14,6 +14,7 @@ module load srw_common module use /lustre/f2/pdata/esrl/gsd/contrib/modulefiles module load rocoto module load cmake/3.20.1 +module load png/1.6.35 setenv CC cc setenv FC ftn From 5c08d6063b24cd40043eb0bce1e77eb65720e0f6 Mon Sep 17 00:00:00 2001 From: Natalie Perlin <68030316+natalie-perlin@users.noreply.github.com> Date: Fri, 10 Jun 2022 03:02:50 -0400 Subject: [PATCH 20/24] Update build_cheyenne_intel Corrected loading system modules (load intel/2022.1, mpt/2.25), updated package versions if newer modules are built. --- modulefiles/build_cheyenne_intel | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/modulefiles/build_cheyenne_intel b/modulefiles/build_cheyenne_intel index e2e25ef9f0..c1d868d458 100644 --- a/modulefiles/build_cheyenne_intel +++ b/modulefiles/build_cheyenne_intel @@ -2,18 +2,16 @@ proc ModulesHelp { } { puts stderr "This module loads libraries for building SRW on" - puts stderr "the CISL Cheyenne machine using Intel-19.1.1" + puts stderr "the CISL Cheyenne machine using Intel-2022.1" } module-whatis "Loads libraries needed for building SRW on Cheyenne" +module purge module load cmake/3.22.0 -module load ncarenv/1.3 -module load intel/2021.2 -module load mpt/2.22 -module load ncarcompilers/0.5.0 +module load intel/2022.1 +module load mpt/2.25 module load python/3.7.9 -module unload netcdf module use /glade/work/epicufsrt/GMTB/tools/intel/2022.1/hpc-stack-v1.2.0_6eb6/modulefiles/stack module load hpc/1.2.0 @@ -21,15 +19,15 @@ module load hpc-intel/2022.1 module load hpc-mpt/2.25 module load srw_common -module load g2/3.4.3 +module load g2/3.4.5 module load esmf/8.3.0b09 module load mapl/2.11.0-esmf-8.3.0b09 module load netcdf/4.7.4 -module load png/1.6.35 -module load pio/2.5.2 +module load libpng/1.6.37 +module load pio/2.5.3 setenv CMAKE_C_COMPILER mpicc -setenv CMAKE_CXX_COMPILER mpicxx +setenv CMAKE_CXX_COMPILER mpixx setenv CMAKE_Fortran_COMPILER mpif90 setenv CMAKE_Platform cheyenne.intel From 7a6d64379f4b63ed48ac0933c5b053c1ee89570b Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Fri, 10 Jun 2022 10:23:30 -0400 Subject: [PATCH 21/24] Updated pointer to model for testing --- Externals.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Externals.cfg b/Externals.cfg index 2458ae6b2b..0f12f5121e 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -4,7 +4,8 @@ protocol = git repo_url = https://github.com/NOAA-EPIC/regional_workflow # Specify either a branch name or a hash but not both. #branch = develop -branch = feature/hera-fix +#branch = release/public-v2 +branch = release-public-v3-SRW local_path = regional_workflow required = True From f6ca015c52698157c1ff4e5393f3dacfe4ef337c Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Fri, 10 Jun 2022 14:36:24 +0000 Subject: [PATCH 22/24] fixed repos --- Externals.cfg | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 0f12f5121e..68b910e592 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,11 +1,10 @@ [regional_workflow] protocol = git -#repo_url = https://github.com/ufs-community/regional_workflow -repo_url = https://github.com/NOAA-EPIC/regional_workflow +repo_url = https://github.com/ufs-community/regional_workflow +#repo_url = https://github.com/NOAA-EPIC/regional_workflow # Specify either a branch name or a hash but not both. #branch = develop -#branch = release/public-v2 -branch = release-public-v3-SRW +branch = release/public-v2 local_path = regional_workflow required = True @@ -20,11 +19,11 @@ required = True [ufs-weather-model] protocol = git -repo_url = https://github.com/ufs-community/ufs-weather-model -#repo_url = https://github.com/NOAA-EPIC/ufs-weather-model-1 +#repo_url = https://github.com/ufs-community/ufs-weather-model +repo_url = https://github.com/NOAA-EPIC/ufs-weather-model-1 # Specify either a branch name or a hash but not both. #branch = develop -hash = release-public-v3-SRW +branch = release-public-v3-SRW local_path = src/ufs-weather-model required = True From af37d2a51964221d31bf4bba5e0f2350aac3803f Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Fri, 10 Jun 2022 12:19:15 -0400 Subject: [PATCH 23/24] updated to point at my regional_workflow branch --- Externals.cfg | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 68b910e592..22654866a1 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,10 +1,11 @@ [regional_workflow] protocol = git -repo_url = https://github.com/ufs-community/regional_workflow -#repo_url = https://github.com/NOAA-EPIC/regional_workflow +#repo_url = https://github.com/ufs-community/regional_workflow +repo_url = https://github.com/NOAA-EPIC/regional_workflow # Specify either a branch name or a hash but not both. #branch = develop -branch = release/public-v2 +#branch = release/public-v2 +branch = feature/hera-fix local_path = regional_workflow required = True From 556af4c8179e03f9ae7aeb153f2b881fd82607f1 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Sun, 12 Jun 2022 18:03:09 -0400 Subject: [PATCH 24/24] updated regional_workflow to release/public-v2 --- Externals.cfg | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Externals.cfg b/Externals.cfg index 22654866a1..7c9e1f1eb2 100644 --- a/Externals.cfg +++ b/Externals.cfg @@ -1,11 +1,9 @@ [regional_workflow] protocol = git -#repo_url = https://github.com/ufs-community/regional_workflow -repo_url = https://github.com/NOAA-EPIC/regional_workflow +repo_url = https://github.com/ufs-community/regional_workflow # Specify either a branch name or a hash but not both. #branch = develop -#branch = release/public-v2 -branch = feature/hera-fix +branch = release/public-v2 local_path = regional_workflow required = True