From 6e2c33a8fca006f76ea508c2e7e2745073331af8 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Wed, 3 Apr 2024 13:42:24 -0600 Subject: [PATCH 01/34] Add a macos clang module file --- scm/etc/modules/macos_clang.lua | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 scm/etc/modules/macos_clang.lua diff --git a/scm/etc/modules/macos_clang.lua b/scm/etc/modules/macos_clang.lua new file mode 100644 index 00000000..71ab2a98 --- /dev/null +++ b/scm/etc/modules/macos_clang.lua @@ -0,0 +1,32 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +a MacOS machine with clang v15.0.0 compilers +]]) + +whatis([===[Loads libraries needed for building the CCPP SCM on a MacOS machine with clang v15.0.0 compilers]===]) + +local ssd=os.getenv("SPACK_STACK_DIR") or LmodError ("Environment variable SPACK_STACK_DIR is not set") +prepend_path("MODULEPATH", ssd .. "/envs/scm-test/install/modulefiles/Core") + +load("stack-apple-clang/15.0.0") +load("stack-python/3.10.13") +load("stack-openmpi/4.1.6") + + +load("cmake/3.28.3") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.1") + +load("py-f90nml/1.4.3") +load("py-netcdf4/1.5.8") + + +load("bacio/2.4.1") +load("sp/2.5.0") +load("w3emc/2.10.0") + +setenv("CMAKE_C_COMPILER","clang") +setenv("CMAKE_CXX_COMPILER","clang++") +setenv("CMAKE_Fortran_COMPILER","gfortran-12") +setenv("CMAKE_Platform","macos.clang") From 1f6014a6120e4b2ef703b1cb374b6848646ee87e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Fri, 5 Apr 2024 09:51:52 -0600 Subject: [PATCH 02/34] Starting documentation updates for build/run - Update Quick Start up through "Compilers" section - Update Repository chapter to include new "Testing" section (needs more expansion later) - Convert section marks to conform with Sphinx Documentation - Fix link and syntax errors --- scm/doc/TechGuide/chap_intro.rst | 2 +- scm/doc/TechGuide/chap_quick.rst | 89 ++++++++++++++------------------ scm/doc/TechGuide/chap_repo.rst | 21 ++++++++ 3 files changed, 60 insertions(+), 52 deletions(-) diff --git a/scm/doc/TechGuide/chap_intro.rst b/scm/doc/TechGuide/chap_intro.rst index 68e4ddf1..94d134d8 100644 --- a/scm/doc/TechGuide/chap_intro.rst +++ b/scm/doc/TechGuide/chap_intro.rst @@ -103,7 +103,7 @@ This release bundle has some known limitations: - As of this release, using the SCM over a land point with an LSM is possible through the use of UFS initial conditions (see - :numref:`Section %s `). However, advective forcing terms + :numref:`Section %s `). However, advective forcing terms are unavailable as of this release, so only short integrations using this configuration should be employed. Using dynamical tendencies (advective forcing terms) from the UFS will be part of a future diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index 69f165b9..e5554b81 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -29,7 +29,7 @@ developer code, but may not be as stable or consistent with existing documentati Instructions for using either option are discussed here. Release Code -~~~~~~~~~~~~ +^^^^^^^^^^^^ Clone the source using @@ -54,7 +54,7 @@ this level. The CCPP physics parameterizations can be found in the .. _`development_code`: Development Code -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ If you would like to contribute as a developer to this project, please see (in addition to the rest of this guide) the scientific and technical @@ -133,62 +133,49 @@ relies on the NetCDF libraries. Beyond the standard scripts, the build system relies on use of the Python scripting language, along with cmake, GNU make and date. -The following software stacks have been tested with this code. Other -versions of various components will likely still work, however. -- gfortran 12.1.0, gcc 12.1.0, cmake 3.23.2, NetCDF 4.7.4, Python - 3.9.12 +For the latest release, the minimum required Python version is 3.8, and CMake requires a minimum version of 3.14. +While exact minimum required versions of other prerequisites have not been established, users can reference the +list of Continuous Integration tests run on the CCPP SCM repository (see :numref:`Section %s `) +for examples of known working configurations. -- GNU compilers 10.1.0, cmake 3.16.4, NetCDF 4.8.1, Python 3.7.12 +Spack-stack +^^^^^^^^^^^^ -- GNU compilers 11.1.0, cmake 3.18.2, NetCDF 4.8.1, Python 3.8.5 +A joint effort between NOAA's Unified Forecast System (UFS) and Joint Effort for Data assimilation Integration (JEDI). +It is designed to be a comprehensive, all-in-one package containing prerequisite libraries and tools needed for all +software in the UFS ecosystem, including the CCPP SCM. As of the version 7, installing spack-stack is the main +supported method of installing the prerequisites needed for building the SCM. The latest version of the SCM is meant +to be built with spack-stack v1.6.0. Older versions may work, but are not guaranteed. Version 1.6.0 of spack-stack +contains the following set of libraries needed for building the SCM: -- Intel compilers 2022.0.2, cmake 3.20.1, NetCDF 4.7.4, Python 3.7.11 + - Netcdf-c (v4.9.2) -- Intel compilers 2022.1.0, cmake 3.22.0, NetCDF 4.8.1, Python 3.7.12 + - Netcdf-FORTRAN (v4.6.0) + + - BACIO (v2.4.1) - Binary I/O Library + + - SP (v2.3.3) - Spectral Transformation Library + + - W3EMC (2.10.0) - GRIB decoder and encoder library + +Instructions for installing spack-stack can be found in the `spack-stack documentation `__. +Spack-stack is already installed and maintained on many HPC platforms, including NSF NCAR's Derecho, NOAA's Hera and +Jet, and MSU's Orion and Hercules. Because these tools are typically the purview of system administrators to install and maintain, they are considered part of the basic system -requirements. The Unified Forecast System (UFS) Short-Range Weather -Application release v1.0.0 of March 2021 provides software packages and -detailed instructions to install these prerequisites and the hpc-stack -on supported platforms (see :numref:`Section %s `) - -Further, there are several utility libraries as part of the hpc-stack -package that must be installed with environment variables pointing to -their locations prior to building the SCM. - -- bacio - Binary I/O Library - -- sp - Spectral Transformation Library - -- w3emc - GRIB decoder and encoder library - -The following environment variables are used by the build system to -properly link these libraries: ``bacio_ROOT``, ``sp_ROOT``, and ``w3emc_ROOT`` Computational platforms on -which these libraries are prebuilt and installed in a central location -are referred to as *preconfigured* platforms. Examples of preconfigured -platforms are most NOAA high-performance computing machines (using the -Intel compiler) and the NCAR Cheyenne system (using the Intel and GNU -compilers). The machine setup scripts mentioned in -:numref:`Section %s ` load these libraries (which are identical -to those used by the UFS Short and Medium Range Weather Applications on -those machines) and set these environment variables for the user -automatically. For installing the libraries and its prerequisites on -supported platforms, existing UFS packages can be used (see -:numref:`Section %s `). +requirements. Compilers -~~~~~~~~~ +^^^^^^^^^ The CCPP and SCM have been tested on a variety of computing platforms. Currently the CCPP system is actively supported on Linux and MacOS computing platforms using the Intel or GNU Fortran compilers. Windows users have a path to use this software through a Docker container that -uses Linux internally (see section `1.5 <#docker>`__). Please use -compiler versions listed in the previous section as unforeseen build -issues may occur when using older versions. Typically the best results -come from using the most recent version of a compiler. If you have +uses Linux internally (see :numref:`Section %s `). Typically the best chance of successfully building and +running the SCM on a new machine comes from using the most recent version of a compiler. If you have problems with compilers, please check the “Known Issues” section of the release website (https://dtcenter.org/community-code/common-community-physics-package-ccpp/download). @@ -196,7 +183,7 @@ release website .. _`use_preconfigured_platforms`: Using Existing Libraries on Preconfigured Platforms -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Because the SCM can be built using the so-called `"spack-stack" libraries `__ @@ -223,7 +210,7 @@ both building and running the SCM. .. _`setup_supported_platforms`: Installing Libraries on Non-preconfigured Platforms -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For users on supported platforms such as generic Linux or macOS systems that have not been preconfigured, the project is suggested for @@ -442,7 +429,7 @@ integrations serially, depending on the command line arguments supplied. .. _`singlerunscript`: Run Script Usage -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ Running a case requires four pieces of information: the case to run (consisting of initial conditions, geolocation, forcing data, etc.), the @@ -518,7 +505,7 @@ below where option abbreviations are included in brackets. value. This is used, for example, in regression testing to reduce total runtimes. -- ``--levels [-l] +- ``--levels [-l]`` - Use this to change the number of vertical levels. @@ -667,7 +654,7 @@ volume-mounting purposes. Any standard NetCDF file viewing or analysis tools may be used to examine the output file (ncdump, ncview, NCL, etc). Batch Run Script -~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^ If using the model on HPC resources and significant amounts of processor time is anticipated for the experiments, it will likely be necessary to @@ -728,7 +715,7 @@ internet search. docker-machine create default --virtualbox-no-vtx-check Building the Docker image -~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^ The Dockerfile builds CCPP SCM v6.0.0 from source using the GNU compiler. A number of required codes are built and installed via the @@ -787,7 +774,7 @@ that was installed as part of Docker Toolbox. docker images Using a prebuilt Docker image from Dockerhub -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ A prebuilt Docker image for this release is available on Dockerhub if it is not desired to build from source. In order to use this, execute the @@ -804,7 +791,7 @@ To verify that it exists afterward, run docker images Running the Docker image -~~~~~~~~~~~~~~~~~~~~~~~~ +^^^^^^^^^^^^^^^^^^^^^^^^ NOTE: Windows users can execute these steps through the Docker Quickstart application installed with Docker Toolbox. diff --git a/scm/doc/TechGuide/chap_repo.rst b/scm/doc/TechGuide/chap_repo.rst index d49b9605..36f0a10c 100644 --- a/scm/doc/TechGuide/chap_repo.rst +++ b/scm/doc/TechGuide/chap_repo.rst @@ -60,3 +60,24 @@ Cubed-Sphere (FV3) dynamical core. | ``│   └── src`` - source code for SCM infrastructure, Python run script, CMakeLists.txt for the SCM, example multirun setup files, suite_info.py | ``└── test`` - Contains scripts for regression testing, Continuous Integration tests +Testing +----------------- + +Regression Testing +^^^^^^^^^^^^^^^^^^ + +Regression tests are a comprehensive set of build and run tests meant to ensure that new changes to the SCM do not break any existing capabilities. These tests are run on code changes before they are merged, and so ensure that the ``main`` branch is always free of major bugs in all facets of the system covered by the tests. + +The latest set of Regression tests are run automatically for every new code change when a Pull Request is opened via GitHub's `Continuous Integration`_. Regression tests are also run manually on a wide variety of platforms in preparation for code release to ensure that all capabilities work as expected for a reasonable spectrum of possible machines a user might want to use. + +Continuous Integration +^^^^^^^^^^^^^^^^^^^^^^ + +The CCPP SCM GitHub repository is set up with Continuous Integration (CI) testing for building the SCM and running some simple test cases. These tests are run automatically on code changes before they are merged, and so ensures that new changes to the SCM do not break basic capabilities. The latest set of tests use the following combinations of SCM prerequisites: + +**Regression tests** + - GNU compilers 11.4.0, Python 3.9.12, netCDF-c 4.7.3, netCDF-FORTRAN 4.5.3, bacio 2.4.1, sp 2.3.3, and w3emc 2.9.2 + +**Build tests** + +All tests use the same versions of NCEP-supported libraries: bacio 2.4.1, sp 2.3.3, and w3emc 2.9.2. Detailed information on these tests can be found in the definition files for these tests, stored in the SCM repository under ``ccpp-scm/.github/workflows``. From 8d90db0fab8438c251bc72b10cb7c146337028c6 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Thu, 25 Apr 2024 13:56:14 -0600 Subject: [PATCH 03/34] Update dockerfile for latest main branch --- docker/Dockerfile | 108 ++++++++++++++++++++++++++++++---------------- 1 file changed, 72 insertions(+), 36 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 99fc2a25..bc78ad72 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,52 +1,88 @@ -FROM dtcenter/common-community-container:gnu9 +FROM debian:12 +MAINTAINER Michael Kavulich -MAINTAINER Michelle Harrold or Grant Firl or Michael Kavulich +# Set up base OS environment -# -# Dockerfile for building CCPP SCM container -# +RUN apt-get -y update -# Note: The common community container image contains the following packages, which are used to build the SCM: -# gfortran, gcc, cmake, netCDF, HDF5, ZLIB, SZIP, Python, and libxml2 -# To access the common community container repository: https://github.com/NCAR/Common-Community-Container +# Get "essential" tools and libraries +RUN apt-get -y install build-essential +RUN apt-get -y install cmake curl git file gfortran-12 ksh m4 python3 tcsh time wget vim +RUN apt-get -y install libnetcdf-pnetcdf-19 libnetcdff7 libnetcdf-dev libnetcdff-dev libxml2 + +MAINTAINER Grant Firl or Michael Kavulich + +#Compiler environment variables +ENV CC /usr/bin/gcc +ENV FC /usr/bin/gfortran +ENV CXX /usr/bin/g++ +ENV F77 /usr/bin/gfortran +ENV F90 /usr/bin/gfortran + +# Other necessary environment variables +ENV LD_LIBRARY_PATH /usr/lib/ + +# Set up unpriviledged user account +RUN groupadd comusers -g 9999 +RUN useradd -u 9999 -g comusers -M -s /bin/bash -c "Unpriviledged user account" -d /home comuser +RUN mkdir /comsoftware \ + && chown -R comuser:comusers /comsoftware \ + && chmod 6755 /comsoftware + +# Set up user home space correctly and make sure user has permissions on all stuff in /comsoftware +RUN chown -R comuser:comusers /home \ + && chmod 6755 /home +RUN chown -R comuser:comusers /comsoftware \ + && chmod -R 6755 /comsoftware + +# Link version-specific aliases +RUN ln -s /usr/bin/python3 /usr/local/bin/python +RUN ln -s /usr/bin/gfortran-12 /usr/bin/gfortran + +# all root steps completed above, now continue below as regular userID comuser +USER comuser +WORKDIR /home + +# Build NCEP libraries we need for SCM + +ENV NCEPLIBS_DIR /comsoftware/nceplibs + +RUN mkdir -p $NCEPLIBS_DIR/src && cd $NCEPLIBS_DIR/src \ + && git clone -b v2.4.1 --recursive https://github.com/NOAA-EMC/NCEPLIBS-bacio \ + && mkdir NCEPLIBS-bacio/build && cd NCEPLIBS-bacio/build \ + && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \ + && make VERBOSE=1 \ + && make install + +RUN cd $NCEPLIBS_DIR/src \ + && git clone -b v2.3.3 --recursive https://github.com/NOAA-EMC/NCEPLIBS-sp \ + && mkdir NCEPLIBS-sp/build && cd NCEPLIBS-sp/build \ + && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \ + && make VERBOSE=1 \ + && make install + +RUN cd $NCEPLIBS_DIR/src \ + && git clone -b v2.11.0 --recursive https://github.com/NOAA-EMC/NCEPLIBS-w3emc \ + && mkdir NCEPLIBS-w3emc/build && cd NCEPLIBS-w3emc/build \ + && cmake -DCMAKE_INSTALL_PREFIX=$NCEPLIBS_DIR .. \ + && make VERBOSE=1 \ + && make install # Obtain CCPP SCM source code RUN cd /comsoftware \ - && git clone --recursive -b release/public-v6 https://github.com/NCAR/ccpp-scm + && git clone --recursive -b main https://github.com/NCAR/ccpp-scm # Obtain static data that was previously stored in repository RUN cd /comsoftware/ccpp-scm/ \ - && . contrib/get_all_static_data.sh + && ./contrib/get_all_static_data.sh # Obtain the pre-computed look-up tables for running with Thompson microphysics RUN cd /comsoftware/ccpp-scm/ \ - && . contrib/get_thompson_tables.sh + && ./contrib/get_thompson_tables.sh -# Run the machine setup script to set environment variables -ENV CC=/opt/rh/devtoolset-9/root/usr/bin/gcc -ENV CXX=/opt/rh/devtoolset-9/root/usr/bin/g++ -ENV F77=/opt/rh/devtoolset-9/root/usr/bin/gfortran -ENV F90=/opt/rh/devtoolset-9/root/usr/bin/gfortran -ENV FC=/opt/rh/devtoolset-9/root/usr/bin/gfortran - -ENV NETCDF=/comsoftware/libs/netcdf - -RUN cd /comsoftware/ccpp-scm/contrib \ - && wget https://raw.githubusercontent.com/NCAR/ccpp-scm/3f501aa8af0fb00ff124d8301c932292d1d0abf3/contrib/build_nceplibs.sh \ - && chmod +x build_nceplibs.sh \ - && cd .. \ - && ./contrib/build_nceplibs.sh $PWD/nceplibs - -ENV bacio_ROOT /comsoftware/ccpp-scm/nceplibs -ENV sp_ROOT /comsoftware/ccpp-scm/nceplibs -ENV w3nco_ROOT /comsoftware/ccpp-scm/nceplibs - -# Create your own link from python -> python3 -# This works without setting the system PATH env var -# since /usr/local/bin is before /usr/bin in the search path. -USER root -RUN ln -s /usr/bin/python3 /usr/local/bin/python -USER comuser +ENV bacio_ROOT /comsoftware/nceplibs +ENV sp_ROOT /comsoftware/nceplibs +ENV w3emc_ROOT /comsoftware/nceplibs # Invoke cmake on the source code to build RUN cd /comsoftware/ccpp-scm/scm \ From e0e18f7f0fd07dfa159d69143423037ceb2bfdf9 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Mon, 29 Apr 2024 09:46:19 -0600 Subject: [PATCH 04/34] Update Dockerfile python setup; now runs test case successfully. Update docs for latest python packages --- docker/Dockerfile | 14 ++++++++++++-- scm/doc/TechGuide/chap_quick.rst | 32 +++++++++----------------------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index bc78ad72..721dda5d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -9,6 +9,7 @@ RUN apt-get -y update RUN apt-get -y install build-essential RUN apt-get -y install cmake curl git file gfortran-12 ksh m4 python3 tcsh time wget vim RUN apt-get -y install libnetcdf-pnetcdf-19 libnetcdff7 libnetcdf-dev libnetcdff-dev libxml2 +RUN apt-get -y install python3-pip python3.11-venv MAINTAINER Grant Firl or Michael Kavulich @@ -35,8 +36,8 @@ RUN chown -R comuser:comusers /home \ RUN chown -R comuser:comusers /comsoftware \ && chmod -R 6755 /comsoftware -# Link version-specific aliases -RUN ln -s /usr/bin/python3 /usr/local/bin/python +# Link version-specific aliases (python3 will be created later with virtual environment) +RUN ln -s ~comuser/.venv/bin/python3 /usr/local/bin/python RUN ln -s /usr/bin/gfortran-12 /usr/bin/gfortran # all root steps completed above, now continue below as regular userID comuser @@ -103,3 +104,12 @@ RUN cd /comsoftware/ccpp-scm/scm \ WORKDIR /comsoftware/ccpp-scm/scm/bin ENV SCM_WORK=/comsoftware ENV SCM_ROOT=/comsoftware/ccpp-scm/ + +# For interactive use, vim mouse settings are infuriating +RUN echo "set mouse=" > ~/.vimrc + +# Set up python virtual environment and install needed packages +ENV VIRTUAL_ENV=~/.venv +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN pip3 install f90nml==1.4.4 netcdf4==1.6.5 diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index e5554b81..1093263d 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -169,7 +169,6 @@ requirements. Compilers ^^^^^^^^^ - The CCPP and SCM have been tested on a variety of computing platforms. Currently the CCPP system is actively supported on Linux and MacOS computing platforms using the Intel or GNU Fortran compilers. Windows @@ -227,44 +226,31 @@ prerequisites (i.e. no ESMF or wgrib2 needed). Users who are not planning to use UFS can install only NetCDF/NetCDF-Fortran manually or using the software package manager (apt, yum, brew). -The Python environment must provide the module for the SCM scripts to -function. Users can test if f90nml is installed using this command in +The Python environment must provide a few non-default modules for the SCM scripts to +function: ``f90nml`` (`documentation `__) and +``netcdf4`` (`documentation `__). Users can test if these are installed using this command in the shell: :: - python -c "import f90nml" + python -c "import f90nml; import netcdf4" If is installed, this command will succeed silently, otherwise an ``ImportError: No module named f90nml`` -will be printed to screen. To install the ``f90nml`` (v0.19) Python module, use the -install method preferred for your Python environment (one of the -following): +will be printed to screen. To install the ``f90nml`` (v1.4.4; ) and ``netcdf4`` (v1.6.5) Python modules, use the +install method preferred for your Python environment (one of the following): - :: - easy_install f90nml==0.19 + easy_install f90nml==1.4.4 netcdf4==1.6.5 - :: - pip install f90nml==0.19 + pip install f90nml==1.4.4 netcdf4==1.6.5 - :: - conda install f90nml=0.19 - -or perform the following steps to install it manually from source: - -:: - - cd /directory/with/write/priveleges - git clone -b v0.19 https://github.com/marshallward/f90nml - cd f90nml - python setup.py install [--prefix=/my/install/directory or --user] + conda install -c conda-forge f90nml==1.4.4 netcdf4==1.6.5 -The directory ``/my/install/directory`` must exist and its subdirectory -``/my/install/directory/lib/python[version]/site-packages`` (or ``lib64`` -instead of ``lib``, depending on the system) must be in the ``PYTHONPATH`` -environment variable. .. _`compiling`: From d1d16a7373916e9c2d2e7d71f6628854c2104cd7 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 09:44:18 -0600 Subject: [PATCH 05/34] Testing out docker autobuild CI --- .github/workflows/ci_docker.yml | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 .github/workflows/ci_docker.yml diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml new file mode 100644 index 00000000..0aedbde2 --- /dev/null +++ b/.github/workflows/ci_docker.yml @@ -0,0 +1,42 @@ +name: build_test_and_push_docker + +on: + push: +# branches: +# - "main" + +env: + TEST_TAG: mkavulich/ccpp-scm:test + LATEST_TAG: mkavulich/ccpp-scm:latest + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and export to Docker + uses: docker/build-push-action@v5 + with: + context: . + load: true + tags: ${{ env.TEST_TAG }} + - name: Test + run: | + docker run --rm -it -v ${OUT_DIR}:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ env.LATEST_TAG }} From b6a1c3167551d1c3f079ad92fcc30c9ba9ea83c7 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 09:56:46 -0600 Subject: [PATCH 06/34] Don't run CI on all pushes --- .github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml index 9e998655..8bb7fced 100644 --- a/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml +++ b/.github/workflows/ci_build_scm_ubuntu_22.04_nvidia.yml @@ -1,6 +1,6 @@ name: CI test to build the CCPP-SCM on ubuntu v22.04 -on: [push,pull_request,workflow_dispatch] +on: [pull_request,workflow_dispatch] jobs: From 5c9b124e5242c807ab52906b86c7f778cc5c27b9 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 10:02:40 -0600 Subject: [PATCH 07/34] Fixing docker context for CI --- .github/workflows/ci_docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 0aedbde2..cc063426 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -28,15 +28,17 @@ jobs: uses: docker/build-push-action@v5 with: context: . + file: docker/Dockerfile load: true tags: ${{ env.TEST_TAG }} - name: Test run: | - docker run --rm -it -v ${OUT_DIR}:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d + docker run --rm -it -v ${{ env.HOME }}:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push uses: docker/build-push-action@v5 with: context: . + file: docker/Dockerfile platforms: linux/amd64,linux/arm64 push: true tags: ${{ env.LATEST_TAG }} From 6b12f31d7c534ec988d590865799ec092b7da65e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 10:23:42 -0600 Subject: [PATCH 08/34] Small users guide addition about Docker usage, try to fix docker CI home space specification --- .github/workflows/ci_docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index cc063426..2a38887b 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -24,7 +24,7 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and export to Docker + - name: Build and export test tag to Docker uses: docker/build-push-action@v5 with: context: . @@ -33,7 +33,7 @@ jobs: tags: ${{ env.TEST_TAG }} - name: Test run: | - docker run --rm -it -v ${{ env.HOME }}:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d + docker run --rm -it -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push uses: docker/build-push-action@v5 with: From cc88611fa939fd09349a3efb99f5815b8c217dcd Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 10:44:28 -0600 Subject: [PATCH 09/34] Testing docker CI --- .github/workflows/ci_docker.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 2a38887b..cdff25f5 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -24,16 +24,16 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and export test tag to Docker - uses: docker/build-push-action@v5 - with: - context: . - file: docker/Dockerfile - load: true - tags: ${{ env.TEST_TAG }} +# - name: Build and export test tag to Docker +# uses: docker/build-push-action@v5 +# with: +# context: . +# file: docker/Dockerfile +# load: true +# tags: ${{ env.TEST_TAG }} - name: Test run: | - docker run --rm -it -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d + docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push uses: docker/build-push-action@v5 with: From edf6b18d2b56777c7ef4614193a195f28c0de9b5 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 10:54:34 -0600 Subject: [PATCH 10/34] Testing push of the test tag to dockerhub --- .github/workflows/ci_docker.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index cdff25f5..a4b90478 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -24,13 +24,14 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} -# - name: Build and export test tag to Docker -# uses: docker/build-push-action@v5 -# with: -# context: . -# file: docker/Dockerfile -# load: true -# tags: ${{ env.TEST_TAG }} + - name: Build and export test tag to Docker + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + load: true + tags: ${{ env.TEST_TAG }} + push: true - name: Test run: | docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d From f12b0b85e497e54602df218d6d1b7ffae02faf97 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 11:21:50 -0600 Subject: [PATCH 11/34] Testing pull of docker CI test tag --- .github/workflows/ci_docker.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index a4b90478..35684fe0 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -24,16 +24,17 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and export test tag to Docker - uses: docker/build-push-action@v5 - with: - context: . - file: docker/Dockerfile - load: true - tags: ${{ env.TEST_TAG }} - push: true +# - name: Build and export test tag to Docker +# uses: docker/build-push-action@v5 +# with: +# context: . +# file: docker/Dockerfile +# load: true +# tags: ${{ env.TEST_TAG }} +# push: true - name: Test run: | + docker pull ${{ env.TEST_TAG }} docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push uses: docker/build-push-action@v5 From 048b01888ef87cfb5196ae1f8bffa93196342ce4 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 11:41:44 -0600 Subject: [PATCH 12/34] Testing HOME permissions for docker CI --- .github/workflows/ci_docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 35684fe0..0f63de47 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -35,6 +35,8 @@ jobs: - name: Test run: | docker pull ${{ env.TEST_TAG }} + echo $HOME + ls -al $HOME docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push uses: docker/build-push-action@v5 From d56352453966428936459c51ac688badf8a429e2 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 11:45:19 -0600 Subject: [PATCH 13/34] Adding write permissions to $HOME for docker CI --- .github/workflows/ci_docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 0f63de47..91bf3f49 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -36,6 +36,7 @@ jobs: run: | docker pull ${{ env.TEST_TAG }} echo $HOME + chmod a+w $HOME ls -al $HOME docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push From 9c8556f395bfcf2cbb19f4ffa3e66cb6045d61bd Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 11:47:50 -0600 Subject: [PATCH 14/34] Adding read+write permissions to $HOME recursively wqfor docker CI --- .github/workflows/ci_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 91bf3f49..b4336a67 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -36,7 +36,7 @@ jobs: run: | docker pull ${{ env.TEST_TAG }} echo $HOME - chmod a+w $HOME + chmod +rw -R $HOME ls -al $HOME docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push From 2d0182e1be79dd19b776bc1e024600dd8ffbdcf8 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 11:50:30 -0600 Subject: [PATCH 15/34] Put output in a subdirectory of $HOME to avoid permissions issues for docker CI --- .github/workflows/ci_docker.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index b4336a67..0530fb5c 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -35,10 +35,10 @@ jobs: - name: Test run: | docker pull ${{ env.TEST_TAG }} - echo $HOME - chmod +rw -R $HOME - ls -al $HOME - docker run --rm -v $HOME:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d + mkdir $HOME/output + chmod +rw -R $HOME/output + ls -al $HOME/output + docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push uses: docker/build-push-action@v5 with: From a0c0c9f7470108ea7d79e770b79917b7ccb72709 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 11:58:39 -0600 Subject: [PATCH 16/34] chmod wasn't working as expected --- .github/workflows/ci_docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 0530fb5c..928159a0 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -36,7 +36,7 @@ jobs: run: | docker pull ${{ env.TEST_TAG }} mkdir $HOME/output - chmod +rw -R $HOME/output + chmod +r +w $HOME/output ls -al $HOME/output docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push From 4f5d74d446d12b353b96edb901de2a1ed6b1b40e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 12:00:32 -0600 Subject: [PATCH 17/34] chmod STILL wasn't working as expected --- .github/workflows/ci_docker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 928159a0..eb9ac5e7 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -36,7 +36,8 @@ jobs: run: | docker pull ${{ env.TEST_TAG }} mkdir $HOME/output - chmod +r +w $HOME/output + chmod +r $HOME/output + chmod +w $HOME/output ls -al $HOME/output docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push From a9bda64b0a5fa377c2270d4b45ef4c1e1801b834 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 12:04:25 -0600 Subject: [PATCH 18/34] I will never ever trust myself with a chmod command again --- .github/workflows/ci_docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index eb9ac5e7..d809ce2a 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -36,8 +36,7 @@ jobs: run: | docker pull ${{ env.TEST_TAG }} mkdir $HOME/output - chmod +r $HOME/output - chmod +w $HOME/output + chmod a+rw $HOME/output ls -al $HOME/output docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d - name: Build and push From 4087a9fb33573ef4eeab127ca0ed339bc35a9d7a Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 12:10:41 -0600 Subject: [PATCH 19/34] It works! Now to try running all test cases --- .github/workflows/ci_docker.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index d809ce2a..8e2be6ee 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -32,13 +32,12 @@ jobs: # load: true # tags: ${{ env.TEST_TAG }} # push: true - - name: Test + - name: Test and push latest tag run: | docker pull ${{ env.TEST_TAG }} mkdir $HOME/output chmod a+rw $HOME/output - ls -al $HOME/output - docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -c twpice -d + docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d - name: Build and push uses: docker/build-push-action@v5 with: From 19350a59fe58c7ac51a0fd447015aa772de3fd00 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 12:14:08 -0600 Subject: [PATCH 20/34] Now to run it all together... --- .github/workflows/ci_docker.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 8e2be6ee..1dd664b5 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -24,14 +24,13 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} -# - name: Build and export test tag to Docker -# uses: docker/build-push-action@v5 -# with: -# context: . -# file: docker/Dockerfile -# load: true -# tags: ${{ env.TEST_TAG }} -# push: true + - name: Build and export test tag to Docker + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + load: true + tags: ${{ env.TEST_TAG }} - name: Test and push latest tag run: | docker pull ${{ env.TEST_TAG }} From e1ce0ccfbfcc8563e62c30bbce913ea2b31e66d3 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 12:21:41 -0600 Subject: [PATCH 21/34] Consolidate dockerfile RUN commands to shrink container size --- .github/workflows/ci_docker.yml | 6 +++--- docker/Dockerfile | 29 +++++++++++------------------ 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_docker.yml index 1dd664b5..65e07d85 100644 --- a/.github/workflows/ci_docker.yml +++ b/.github/workflows/ci_docker.yml @@ -24,20 +24,20 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and export test tag to Docker + - name: Build and export test tag uses: docker/build-push-action@v5 with: context: . file: docker/Dockerfile load: true tags: ${{ env.TEST_TAG }} - - name: Test and push latest tag + - name: Test run: | docker pull ${{ env.TEST_TAG }} mkdir $HOME/output chmod a+rw $HOME/output docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d - - name: Build and push + - name: Push latest tag uses: docker/build-push-action@v5 with: context: . diff --git a/docker/Dockerfile b/docker/Dockerfile index 721dda5d..9699254c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,10 +6,10 @@ MAINTAINER Michael Kavulich RUN apt-get -y update # Get "essential" tools and libraries -RUN apt-get -y install build-essential -RUN apt-get -y install cmake curl git file gfortran-12 ksh m4 python3 tcsh time wget vim -RUN apt-get -y install libnetcdf-pnetcdf-19 libnetcdff7 libnetcdf-dev libnetcdff-dev libxml2 -RUN apt-get -y install python3-pip python3.11-venv +RUN apt-get -y install build-essential \ + && apt-get -y install cmake curl git file gfortran-12 ksh m4 python3 tcsh time wget vim \ + && apt-get -y install libnetcdf-pnetcdf-19 libnetcdff7 libnetcdf-dev libnetcdff-dev libxml2 \ + && apt-get -y install python3-pip python3.11-venv MAINTAINER Grant Firl or Michael Kavulich @@ -69,24 +69,17 @@ RUN cd $NCEPLIBS_DIR/src \ && make VERBOSE=1 \ && make install -# Obtain CCPP SCM source code -RUN cd /comsoftware \ - && git clone --recursive -b main https://github.com/NCAR/ccpp-scm - -# Obtain static data that was previously stored in repository -RUN cd /comsoftware/ccpp-scm/ \ - && ./contrib/get_all_static_data.sh - -# Obtain the pre-computed look-up tables for running with Thompson microphysics -RUN cd /comsoftware/ccpp-scm/ \ - && ./contrib/get_thompson_tables.sh - ENV bacio_ROOT /comsoftware/nceplibs ENV sp_ROOT /comsoftware/nceplibs ENV w3emc_ROOT /comsoftware/nceplibs -# Invoke cmake on the source code to build -RUN cd /comsoftware/ccpp-scm/scm \ +# Obtain CCPP SCM source code and static data, build code +RUN cd /comsoftware \ + && git clone --recursive -b main https://github.com/NCAR/ccpp-scm \ + && cd /comsoftware/ccpp-scm/ \ + && ./contrib/get_all_static_data.sh \ + && ./contrib/get_thompson_tables.sh \ + && cd /comsoftware/ccpp-scm/scm \ && mkdir bin \ && cd bin \ && cmake ../src \ From 78ad73c510bf153e4a68b9bb5e5133c16e01fe98 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 20:45:03 -0600 Subject: [PATCH 22/34] More updates to Quick Start chapter: - remove a lot of fluff to get to the details quicker - Add example commands for installing spack-stack on MacOS - Update "multi-run" command for Docker Now updated through "4.3. Compiling SCM with CCPP" --- .../{ci_docker.yml => ci_test_docker.yml} | 0 scm/doc/TechGuide/chap_quick.rst | 129 ++++++++++++------ 2 files changed, 84 insertions(+), 45 deletions(-) rename .github/workflows/{ci_docker.yml => ci_test_docker.yml} (100%) diff --git a/.github/workflows/ci_docker.yml b/.github/workflows/ci_test_docker.yml similarity index 100% rename from .github/workflows/ci_docker.yml rename to .github/workflows/ci_test_docker.yml diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index 1093263d..c54ff8f8 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -4,18 +4,8 @@ Quick Start Guide ================= This chapter provides instructions for obtaining and compiling the CCPP -SCM. The SCM code calls CCPP-compliant physics schemes through the CCPP -framework code. As such, it requires the CCPP framework code and physics -code, both of which are included as submodules within the SCM git repository. This -package can be considered a simple example for an atmospheric model to -interact with physics through the CCPP. - -Alternatively, if one doesn’t have access to or care to set up a machine -with the appropriate system requirements but has a working Docker -installation, it is possible to create and use a Docker container with a -pre-configured computing environment with a pre-compiled model. This is -also an avenue for running this software with a Windows PC. See section -:numref:`Section %s ` for more information. +SCM. We provide instructions on building the code from scratch (:numref:`Section %s `), as well as +using Docker containers for machines that have Docker software installed (:numref:`Section %s `). .. _obtaining_code: @@ -37,11 +27,10 @@ Clone the source using git clone --recursive -b v6.0.0 https://github.com/NCAR/ccpp-scm -By using the ``--recursive`` option, it guarantees that you are checking out the commits -of ccpp-physics and ccpp-framework that were tested with the latest -commit of the SCM main branch. If not included initially, you can always retrieve the commits of -the submodules that were intended to be used with a given commit of the -SCM by executing the following command from the SCM directory: +The ``--recursive`` option is required to retrieve the ccpp-physics and ccpp-framework code, +which are stored in separate repositories and linked to the SCM repository as submodules. +If not included initially, you can always retrieve the submodules +by executing the following command from the SCM directory: .. code:: bash @@ -56,17 +45,8 @@ this level. The CCPP physics parameterizations can be found in the Development Code ^^^^^^^^^^^^^^^^ -If you would like to contribute as a developer to this project, please -see (in addition to the rest of this guide) the scientific and technical -documentation included with this release for both the SCM and the CCPP: - -https://dtcenter.org/community-code/common-community-physics-package-ccpp/documentation - -There you will find links to all of the documentation pertinent to -developers. - -For working with the development branches (stability not guaranteed), -check out the ``main`` branch of the repository: +Developers seeking to contribute code to the SCM or CCPP will need to use the most up-to-date +version of the code, which can be found on the ``main`` branch of the repository: .. code:: bash @@ -82,7 +62,21 @@ SCM by executing the following command from the SCM directory: git submodule update --init --recursive -You can try to use the latest commits of the ccpp-physics and +While the ``main`` branch is tested regularly for compilation and basic functionality (as described in :numref:`Section %s `), +it may not be as stable or scientifically vetted as the latest release code, and may be lacking in up-to-date documentation. + +If you would like to contribute as a developer to this project, please +see (in addition to the rest of this guide) the scientific and technical +documentation included with this release for both the SCM and the CCPP: + +https://dtcenter.org/community-code/common-community-physics-package-ccpp/documentation + +There you will find links to all of the documentation pertinent to +developers. + + +While the SCM is updated with the latest commits to the CCPP submodules (ccpp-physics and ccpp-framework) +on a fairly regular basis, it may be weeks or even months behind at times. You can try to use the latest commits of the ccpp-physics and ccpp-framework submodules if you wish, but this may not have been tested (i.e. SCM development may lag ccpp-physics and/or ccpp-framework development). To do so: @@ -129,11 +123,12 @@ System Requirements, Libraries, and Tools The source code for the SCM and CCPP components is in the form of programs written in FORTRAN 90 (with some required features from the FORTRAN 2008 standard), and C. In addition, the model I/O -relies on the NetCDF libraries. Beyond the standard scripts, the build +relies on the NetCDF libraries, as well as the NCEP libraries ``bacio``, ``sp`` and ``w3emc``. + +Beyond the standard shell scripts, the build system relies on use of the Python scripting language, along with cmake, GNU make and date. - For the latest release, the minimum required Python version is 3.8, and CMake requires a minimum version of 3.14. While exact minimum required versions of other prerequisites have not been established, users can reference the list of Continuous Integration tests run on the CCPP SCM repository (see :numref:`Section %s `) @@ -163,10 +158,6 @@ Instructions for installing spack-stack can be found in the `spack-stack documen Spack-stack is already installed and maintained on many HPC platforms, including NSF NCAR's Derecho, NOAA's Hera and Jet, and MSU's Orion and Hercules. -Because these tools are typically the purview of system administrators -to install and maintain, they are considered part of the basic system -requirements. - Compilers ^^^^^^^^^ The CCPP and SCM have been tested on a variety of computing platforms. @@ -212,21 +203,69 @@ Installing Libraries on Non-preconfigured Platforms ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ For users on supported platforms such as generic Linux or macOS systems -that have not been preconfigured, the project is suggested for -installing prerequisite libraries. Visit -https://github.com/NOAA-EMC/hpc-stack for instructions for installing -prerequisite libraries via *hpc-stack* in their docs directory. UFS users who -already installed libraries via the *hpc-stack* package only need to set the -compiler (``CC``, ``CXX``, ``FC``), NetCDF (``NetCDF_ROOT``), and ``bacio``, -``sp`` and ``w3emc`` (``bacio_ROOT``, ``sp_ROOT``, ``w3emc_ROOT``) environment variables to point -to their installation paths in order to compile the SCM. +that have not been preconfigured, installing ``spack-stack`` (see :ref:`Section %s `) +is highly recommended, as it provides all the necessary prerequisite libraries needed for installing the SCM. + + +MacOS example +"""""""""""""" + +As an example of the steps needed for installing spack-stack on a MacOS machine, here is the list of steps that +resulted in a successful build and run of the SCM on a MacBook Pro with the Ventura 13.6.3 operating system, +using ``clang`` compilers. The necessary prerequisites for spack-stack (see +`spack-stack documentation `__) +were installed using Homebrew. + +:: + + cd your_experiment_dir + export TOP_DIR=`pwd` + mkdir spack_stack_1.6.0 + cd spack_stack_1.6.0 + git clone --recurse-submodules https://github.com/jcsda/spack-stack.git -b 1.6.0 + cd spack-stack + source setup.sh + ulimit -S -s unlimited + spack stack create env --site macos.default --template ufs-weather-model --name SCM + cd envs/SCM + spack env activate -p . + export SPACK_SYSTEM_CONFIG_PATH="$PWD/site" + spack external find --scope system --exclude bison --exclude openssl --exclude python + spack external find --scope system libiconv + spack external find --scope system perl + spack external find --scope system wget + PATH="$HOMEBREW_ROOT/opt/curl/bin:$PATH" spack external find --scope system curl + PATH="$HOMEBREW_ROOT/opt/qt5/bin:$PATH" spack external find --scope system qt + spack config --scope system add packages:pkg-config:buildable:false + spack compiler find --scope system + unset SPACK_SYSTEM_CONFIG_PATH + spack config add "packages:all:compiler:[apple-clang@15.0.0]" + spack config add "packages:all:providers:mpi:[openmpi@4.1.6]" + spack concretize 2>&1 | tee log.concretize + ../../util/show_duplicate_packages.py -d log.concretize + spack install --verbose --fail-fast 2>&1 | tee log.install + spack module lmod refresh + spack stack setup-meta-modules + + +Setting up compilation environment +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Once libraries are installed, either with spack-stack or manually, users will need to set some environment variables +needed for specifying the location of the various prerequisites. Users will need to set variables for the +compilers (``CC``, ``CXX``, ``FC``), as well as the root directories for the library installs of NetCDF (``NetCDF_ROOT``), +``bacio`` (``bacio_ROOT``), ``sp`` (``sp_ROOT``), and ``w3emc`` (``w3emc_ROOT``). The SCM uses only a small part of the UFS *hpc-stack* package and has fewer prerequisites (i.e. no ESMF or wgrib2 needed). Users who are not planning to use the UFS can install only NetCDF/NetCDF-Fortran manually or using the software package manager (apt, yum, brew). -The Python environment must provide a few non-default modules for the SCM scripts to +Python requirements +""""""""""""""""""""" + +The SCM build system invokes the ``ccpp_prebuild.py`` script, and so the Python environment must be set up prior to building. +As mentioned earlier, a minimum Python version of 3.8 is required. Additionally, there are a few non-default modules required for the SCM to function: ``f90nml`` (`documentation `__) and ``netcdf4`` (`documentation `__). Users can test if these are installed using this command in the shell: @@ -837,7 +876,7 @@ Quickstart application installed with Docker Toolbox. .. code:: bash - docker run --rm -it -v ${OUT_DIR}:/home --name run-ccpp-scm ccpp-scm ./run_scm.py -m -d + docker run --rm -it -v ${OUT_DIR}:/home --name run-ccpp-scm ccpp-scm ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d The options included in the above ``run`` commands are the following: From 1ff40ce2da77749e70b175ad0d11a125bdce4374 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Tue, 30 Apr 2024 20:48:11 -0600 Subject: [PATCH 23/34] Split out the CI test that pushes new container to Dockerhub, only run this on pushes to main --- .../build_and_push_docker_latest.yml | 34 +++++++++++++++++++ .github/workflows/ci_test_docker.yml | 17 ++-------- 2 files changed, 36 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/build_and_push_docker_latest.yml diff --git a/.github/workflows/build_and_push_docker_latest.yml b/.github/workflows/build_and_push_docker_latest.yml new file mode 100644 index 00000000..d9dd7319 --- /dev/null +++ b/.github/workflows/build_and_push_docker_latest.yml @@ -0,0 +1,34 @@ +name: build_test_and_push_docker + +on: + push: + branches: + # Only build containers when pushing to main + - "main" + +env: + LATEST_TAG: NCAR/ccpp-scm:latest + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Push latest tag + uses: docker/build-push-action@v5 + with: + context: . + file: docker/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ env.LATEST_TAG }} diff --git a/.github/workflows/ci_test_docker.yml b/.github/workflows/ci_test_docker.yml index 65e07d85..1fac0d69 100644 --- a/.github/workflows/ci_test_docker.yml +++ b/.github/workflows/ci_test_docker.yml @@ -1,13 +1,9 @@ name: build_test_and_push_docker -on: - push: -# branches: -# - "main" +on: [pull_request,workflow_dispatch] env: - TEST_TAG: mkavulich/ccpp-scm:test - LATEST_TAG: mkavulich/ccpp-scm:latest + TEST_TAG: NCAR/ccpp-scm:test jobs: docker: @@ -33,15 +29,6 @@ jobs: tags: ${{ env.TEST_TAG }} - name: Test run: | - docker pull ${{ env.TEST_TAG }} mkdir $HOME/output chmod a+rw $HOME/output docker run --rm -v $HOME/output:/home ${{ env.TEST_TAG }} ./run_scm.py -f ../../test/rt_test_cases.py --runtime_mult 0.1 -d - - name: Push latest tag - uses: docker/build-push-action@v5 - with: - context: . - file: docker/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ env.LATEST_TAG }} From 3d2ce1bf067cee32dbc3b09ffcd858bf43036bde Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 10:56:57 -0600 Subject: [PATCH 24/34] Add Linux GNU modulefile, remove compiler version from macos_clang modulefile for flexibility --- scm/etc/modules/linux_gnu.lua | 33 +++++++++++++++++++++++++++++++++ scm/etc/modules/macos_clang.lua | 6 +++--- 2 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 scm/etc/modules/linux_gnu.lua diff --git a/scm/etc/modules/linux_gnu.lua b/scm/etc/modules/linux_gnu.lua new file mode 100644 index 00000000..1c0c65ef --- /dev/null +++ b/scm/etc/modules/linux_gnu.lua @@ -0,0 +1,33 @@ +help([[ +This module loads libraries for building the CCPP Single-Column Model on +a generic Linux machine with GNU compilers. Note that you may have to make +modifications to this file based on your own compilers and specifics of +how spack-stack was installed; see the Users Guide for details +]]) + +whatis([===[Loads libraries needed for building the CCPP SCM on a Linux machine with GNU compilers]===]) + +local ssd=os.getenv("SPACK_STACK_DIR") or LmodError ("Environment variable SPACK_STACK_DIR is not set") +prepend_path("MODULEPATH", ssd .. "/envs/scm-test/install/modulefiles/Core") + +load("stack-gnu") +load("stack-python/3.10.13") +load("stack-openmpi/4.1.6") + + +load("cmake/3.28.3") + +load("netcdf-c/4.9.2") +load("netcdf-fortran/4.6.1") + +load("py-f90nml/1.4.3") +load("py-netcdf4/1.5.8") + + +load("bacio/2.4.1") +load("sp/2.5.0") +load("w3emc/2.10.0") + +setenv("CMAKE_C_COMPILER","mpicc") +setenv("CMAKE_CXX_COMPILER","mpicxx") +setenv("CMAKE_Fortran_COMPILER","mpif90") diff --git a/scm/etc/modules/macos_clang.lua b/scm/etc/modules/macos_clang.lua index 71ab2a98..41aef768 100644 --- a/scm/etc/modules/macos_clang.lua +++ b/scm/etc/modules/macos_clang.lua @@ -1,14 +1,14 @@ help([[ This module loads libraries for building the CCPP Single-Column Model on -a MacOS machine with clang v15.0.0 compilers +a MacOS machine with clang compilers ]]) -whatis([===[Loads libraries needed for building the CCPP SCM on a MacOS machine with clang v15.0.0 compilers]===]) +whatis([===[Loads libraries needed for building the CCPP SCM on a MacOS machine with clang compilers]===]) local ssd=os.getenv("SPACK_STACK_DIR") or LmodError ("Environment variable SPACK_STACK_DIR is not set") prepend_path("MODULEPATH", ssd .. "/envs/scm-test/install/modulefiles/Core") -load("stack-apple-clang/15.0.0") +load("stack-apple-clang") load("stack-python/3.10.13") load("stack-openmpi/4.1.6") From a93a5a527103e2c74239be6372f68f5f0c5bcc76 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 11:02:16 -0600 Subject: [PATCH 25/34] More quick-start updates - Move MacOS example out to an external link (Github Discussion) - Move downloading of static data after build - Convert "NOTE" asides into proper Sphinx Note blocks - More details about debug mode - Remove yet another forum mention, how are these still out there!? --- scm/doc/TechGuide/chap_quick.rst | 169 +++++++++++++++---------------- 1 file changed, 84 insertions(+), 85 deletions(-) diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index c54ff8f8..e5be962a 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -205,47 +205,14 @@ Installing Libraries on Non-preconfigured Platforms For users on supported platforms such as generic Linux or macOS systems that have not been preconfigured, installing ``spack-stack`` (see :ref:`Section %s `) is highly recommended, as it provides all the necessary prerequisite libraries needed for installing the SCM. +The main downside to spack-stack is that it contains a large number of libraries and utilities used by the whole +Unified Forecast System and related applications, only a minority of which are required for the SCM. Users may +install libraries manually if they wish, but they will need to make sure the appropriate environment variables +are set to the correct values so that the build system can find them: - -MacOS example -"""""""""""""" - -As an example of the steps needed for installing spack-stack on a MacOS machine, here is the list of steps that -resulted in a successful build and run of the SCM on a MacBook Pro with the Ventura 13.6.3 operating system, -using ``clang`` compilers. The necessary prerequisites for spack-stack (see -`spack-stack documentation `__) -were installed using Homebrew. - -:: - - cd your_experiment_dir - export TOP_DIR=`pwd` - mkdir spack_stack_1.6.0 - cd spack_stack_1.6.0 - git clone --recurse-submodules https://github.com/jcsda/spack-stack.git -b 1.6.0 - cd spack-stack - source setup.sh - ulimit -S -s unlimited - spack stack create env --site macos.default --template ufs-weather-model --name SCM - cd envs/SCM - spack env activate -p . - export SPACK_SYSTEM_CONFIG_PATH="$PWD/site" - spack external find --scope system --exclude bison --exclude openssl --exclude python - spack external find --scope system libiconv - spack external find --scope system perl - spack external find --scope system wget - PATH="$HOMEBREW_ROOT/opt/curl/bin:$PATH" spack external find --scope system curl - PATH="$HOMEBREW_ROOT/opt/qt5/bin:$PATH" spack external find --scope system qt - spack config --scope system add packages:pkg-config:buildable:false - spack compiler find --scope system - unset SPACK_SYSTEM_CONFIG_PATH - spack config add "packages:all:compiler:[apple-clang@15.0.0]" - spack config add "packages:all:providers:mpi:[openmpi@4.1.6]" - spack concretize 2>&1 | tee log.concretize - ../../util/show_duplicate_packages.py -d log.concretize - spack install --verbose --fail-fast 2>&1 | tee log.install - spack module lmod refresh - spack stack setup-meta-modules +The CCPP/SCM team does not support spack-stack, so users with questions or requiring help with spack-stack installation +should reference the spack-stack documentation. However, we have provided an example procedure in +`this GitHub discussion `__. Setting up compilation environment @@ -254,12 +221,26 @@ Setting up compilation environment Once libraries are installed, either with spack-stack or manually, users will need to set some environment variables needed for specifying the location of the various prerequisites. Users will need to set variables for the compilers (``CC``, ``CXX``, ``FC``), as well as the root directories for the library installs of NetCDF (``NetCDF_ROOT``), -``bacio`` (``bacio_ROOT``), ``sp`` (``sp_ROOT``), and ``w3emc`` (``w3emc_ROOT``). +``bacio`` (``bacio_ROOT``), ``sp`` (``sp_ROOT``), and ``w3emc`` (``w3emc_ROOT``). This is the procedure used in the +provided Dockerfile in ``ccpp-scm/docker/``, so users can reference that file for guidance on how to install this software +and set these variables. + +For users on a pre-configured platform, you can load the spack-stack environment via one of the provided modules in ``scm/etc/modules/``. +For example, users on the NSF NCAR machine Derecho who wish to use Intel compilers can do the following: + +:: + + cd [path/to/ccpp-scm/] + module use scm/etc/modules/ + module load derecho_intel + +Additionally, for users who have installed spack-stack on their own MacOS or Linux machine can use the provided ``macos_clang`` +or ``linux_gnu`` modules. + +.. note:: -The SCM uses only a small part of the UFS *hpc-stack* package and has fewer -prerequisites (i.e. no ESMF or wgrib2 needed). Users who are not planning to use the -UFS can install only NetCDF/NetCDF-Fortran manually or using the -software package manager (apt, yum, brew). + The provided modules assume ``clang``/``gfortran`` compilers on MacOS and GNU compilers for Linux. + If you are using a different set of compilers, you may need to modify the module file. Python requirements """"""""""""""""""""" @@ -298,24 +279,7 @@ Compiling SCM with CCPP The first step in compiling the CCPP and SCM is to properly setup your user environment as described in -sections :numref:`%s ` and :numref:`Section %s `. The second step is -to download the lookup tables and other large datasets (large binaries, -:math:`<`\ 1 GB) needed by the physics schemes and place them in the -correct directory: From the top-level code directory (``ccpp-scm`` by default), -execute the following scripts: - -.. code:: bash - - ./contrib/get_all_static_data.sh - ./contrib/get_thompson_tables.sh - -If the download step fails, make sure that your system’s firewall does -not block access to GitHub. If it does, download the files ``comparison_data.tar.gz``, -``physics_input_data.tar.gz``, ``processed_case_input.tar.gz``, and ``raw_case_input.tar.gz`` -from the GitHub release website using your browser and manually extract its -contents in the directory ``scm/data``. Similarly, do the same for -``thompson_tables.tar.gz`` and ``MG_INCCN_data.tar.gz`` and extract -to ``scm/data/physics_input_data/``. +sections :numref:`%s ` and :numref:`Section %s `. Following this step, the top level build system will use ``cmake`` to query system parameters, execute the CCPP prebuild script to match the physics @@ -379,7 +343,8 @@ components. -DOPENMP=ON - - Debug mode + - Debug mode, which compiles with lower optimization and additional compile-time checks. Only + recommended for development and debugging, because code compiled in this mode will run slower. .. code:: bash @@ -433,12 +398,13 @@ directory) pwd #confirm that you are in the ccpp-scm/scm/bin directory before deleting files rm -rfd * -Note: This command can be dangerous (deletes files without confirming), -so make sure that you’re in the right directory before executing! +.. note:: + This command can be dangerous (deletes files without confirming), + so make sure that you’re in the right directory before executing! If you encounter errors, please capture a log file from all of the -steps, and start a thread on the support forum at: -https://dtcenter.org/forum/ccpp-user-support/ccpp-single-column-model +steps, and start a thread on the Github Discussions support forum at: +https://github.com/NCAR/ccpp-scm/discussions Run the SCM with a supplied case -------------------------------- @@ -451,6 +417,27 @@ executed through a Python run script that is pre-staged into the ``bin`` directory: ``run_scm.py``. It can be used to run one integration or several integrations serially, depending on the command line arguments supplied. +Downloading input data +^^^^^^^^^^^^^^^^^^^^^^ +The various SCM cases require staged input data in order to run. This includes +input data for cases and lookup tables for runtime use. This is a large dataset +(:math:`<`\ 1 GB) so it is not stored in the SCM repository, and must be downloaded +separately. To download this data place it in the correct directories, +execute the following scripts: + +.. code:: bash + + ./contrib/get_all_static_data.sh + ./contrib/get_thompson_tables.sh + +If the download step fails, make sure that your system’s firewall does +not block access to GitHub. If it does, download the files ``comparison_data.tar.gz``, +``physics_input_data.tar.gz``, ``processed_case_input.tar.gz``, and ``raw_case_input.tar.gz`` +from the `SCM release page `__ using your browser and manually extract its +contents in the directory ``scm/data``. Similarly, do the same for +``thompson_tables.tar.gz`` and ``MG_INCCN_data.tar.gz`` and extract +to ``scm/data/physics_input_data/``. + .. _`singlerunscript`: Run Script Usage @@ -709,10 +696,12 @@ In order to run a precompiled version of the CCPP SCM in a container, Docker will need to be available on your machine. Please visit https://www.docker.com to download and install the version compatible with your system. Docker frequently releases updates to the software; it -is recommended to apply all available updates. NOTE: In order to install -Docker on your machine, you will be required to have root access -privileges. More information about getting started can be found at -https://docs.docker.com/get-started +is recommended to apply all available updates. + +.. note:: + In order to install Docker on your machine, you will be required to have root access + privileges. More information about getting started can be found at + https://docs.docker.com/get-started The following tips were acquired during a recent installation of Docker on a machine with Windows 10 Home Edition. Further help should be @@ -775,8 +764,9 @@ can be generated from the code in the software repository obtained by following the instructions in :numref:`Section %s `, and then executing the following steps: -NOTE: Windows users can execute these steps in the terminal application -that was installed as part of Docker Toolbox. +.. note:: + Windows users can execute these steps in the terminal application + that was installed as part of Docker Toolbox. #. Navigate to the ``ccpp-scm/docker`` directory. @@ -818,8 +808,9 @@ To verify that it exists afterward, run Running the Docker image ^^^^^^^^^^^^^^^^^^^^^^^^ -NOTE: Windows users can execute these steps through the Docker -Quickstart application installed with Docker Toolbox. +.. note:: + Windows users can execute these steps through the Docker + Quickstart application installed with Docker Toolbox. #. Set up a directory that will be shared between the host machine and the Docker container. When set up correctly, it will contain output @@ -870,9 +861,13 @@ Quickstart application installed with Docker Toolbox. docker run --rm -it -v ${OUT_DIR}:/home --name run-ccpp-scm ccpp-scm ./run_scm.py -c twpice -d will run through the TWPICE case using the default suite and namelist - and put the output in the shared directory. NOTE: Windows users may - need to omit the curly braces around environment variables: use ``$OUT_DIR`` - instead of ``${OUT_DIR}``. For running through all supported cases and suites, use + and put the output in the shared directory. + + .. note:: + Windows users may need to omit the curly braces around environment variables: use ``$OUT_DIR`` + instead of ``${OUT_DIR}``. + + For running through all supported cases and suites, use .. code:: bash @@ -894,9 +889,10 @@ Quickstart application installed with Docker Toolbox. - ``−−name`` names the container. If no name is provided, the daemon will autogenerate a random string name. - NOTE: If you are using a prebuilt image from Dockerhub, substitute - the name of the image that was pulled from Dockerhub in the commands - above; i.e. instead of ``ccpp-scm`` above, one would have ``dtcenter/ccpp-scm:v6.0.0``. + .. note:: + If you are using a prebuilt image from Dockerhub, substitute + the name of the image that was pulled from Dockerhub in the commands + above; i.e. instead of ``ccpp-scm`` above, one would have ``dtcenter/ccpp-scm:v6.0.0``. #. To use the SCM interactively, run non-default configurations, create plots, or even develop code, issue the following command: @@ -909,6 +905,9 @@ Quickstart application installed with Docker Toolbox. directory of the SCM with a pre-compiled executable. At this point, one could use the run scripts as described in previous sections (remembering to include the option on run scripts if output is to be - shared with the host machine). NOTE: If developing, since the - container is ephemeral, one should push their changes to a remote git - repository to save them (i.e. a fork on GitHub.com). + shared with the host machine). + + .. note:: + + If developing or modifying code, since the container is ephemeral, one should push their changes to a remote git + repository to save them (i.e. a fork on GitHub.com). From 18fb473e22f2de3513ac61a2c08695e50ce62d1d Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 15:51:21 -0600 Subject: [PATCH 26/34] Minor users guide updates --- scm/doc/TechGuide/chap_quick.rst | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index e5be962a..6b77a944 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -156,7 +156,7 @@ contains the following set of libraries needed for building the SCM: Instructions for installing spack-stack can be found in the `spack-stack documentation `__. Spack-stack is already installed and maintained on many HPC platforms, including NSF NCAR's Derecho, NOAA's Hera and -Jet, and MSU's Orion and Hercules. +Jet, and MSU's Orion. Compilers ^^^^^^^^^ @@ -205,26 +205,21 @@ Installing Libraries on Non-preconfigured Platforms For users on supported platforms such as generic Linux or macOS systems that have not been preconfigured, installing ``spack-stack`` (see :ref:`Section %s `) is highly recommended, as it provides all the necessary prerequisite libraries needed for installing the SCM. -The main downside to spack-stack is that it contains a large number of libraries and utilities used by the whole -Unified Forecast System and related applications, only a minority of which are required for the SCM. Users may -install libraries manually if they wish, but they will need to make sure the appropriate environment variables -are set to the correct values so that the build system can find them: The CCPP/SCM team does not support spack-stack, so users with questions or requiring help with spack-stack installation -should reference the spack-stack documentation. However, we have provided an example procedure in +should reference the `spack-stack documentation `__. +However, we have provided an example procedure in `this GitHub discussion `__. +The main downside to spack-stack is that it contains a large number of libraries and utilities used by the whole +Unified Forecast System and related applications, only a minority of which are required for the SCM. Users may +install libraries manually if they wish, but they will need to make sure the appropriate environment variables +are set to the correct values so that the build system can find them, as described in the following chapter. + Setting up compilation environment ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Once libraries are installed, either with spack-stack or manually, users will need to set some environment variables -needed for specifying the location of the various prerequisites. Users will need to set variables for the -compilers (``CC``, ``CXX``, ``FC``), as well as the root directories for the library installs of NetCDF (``NetCDF_ROOT``), -``bacio`` (``bacio_ROOT``), ``sp`` (``sp_ROOT``), and ``w3emc`` (``w3emc_ROOT``). This is the procedure used in the -provided Dockerfile in ``ccpp-scm/docker/``, so users can reference that file for guidance on how to install this software -and set these variables. - For users on a pre-configured platform, you can load the spack-stack environment via one of the provided modules in ``scm/etc/modules/``. For example, users on the NSF NCAR machine Derecho who wish to use Intel compilers can do the following: @@ -242,6 +237,14 @@ or ``linux_gnu`` modules. The provided modules assume ``clang``/``gfortran`` compilers on MacOS and GNU compilers for Linux. If you are using a different set of compilers, you may need to modify the module file. +If libraries were installed manually, users will need to set some environment variables +needed for specifying the location of the various prerequisites. Users will need to set variables for the +compilers (``CC``, ``CXX``, ``FC``), as well as the root directories for the library installs of NetCDF (``NetCDF_ROOT``), +``bacio`` (``bacio_ROOT``), ``sp`` (``sp_ROOT``), and ``w3emc`` (``w3emc_ROOT``). This is the procedure used in the +provided Dockerfile in ``ccpp-scm/docker/``, so users can reference that file for guidance on how to install this software +and set these variables. + + Python requirements """"""""""""""""""""" From daf85633e006387b3f4f8b8860f289060bdfb09e Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Wed, 1 May 2024 15:52:32 -0600 Subject: [PATCH 27/34] Consolidate some repeated directory setup commands in Dockerfile --- docker/Dockerfile | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 9699254c..6212a051 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -23,17 +23,13 @@ ENV F90 /usr/bin/gfortran # Other necessary environment variables ENV LD_LIBRARY_PATH /usr/lib/ -# Set up unpriviledged user account -RUN groupadd comusers -g 9999 -RUN useradd -u 9999 -g comusers -M -s /bin/bash -c "Unpriviledged user account" -d /home comuser -RUN mkdir /comsoftware \ - && chown -R comuser:comusers /comsoftware \ - && chmod 6755 /comsoftware - -# Set up user home space correctly and make sure user has permissions on all stuff in /comsoftware -RUN chown -R comuser:comusers /home \ +# Set up unpriviledged user account, set up user home space and make sure user has permissions on all stuff in /comsoftware +RUN groupadd comusers -g 9999 \ + && useradd -u 9999 -g comusers -M -s /bin/bash -c "Unpriviledged user account" -d /home comuser \ + && mkdir /comsoftware \ + && chown -R comuser:comusers /home \ && chmod 6755 /home -RUN chown -R comuser:comusers /comsoftware \ + && chown -R comuser:comusers /comsoftware \ && chmod -R 6755 /comsoftware # Link version-specific aliases (python3 will be created later with virtual environment) From 755e0c51f8e8ffcb6a264056cc5b9cf617b54848 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr." Date: Wed, 1 May 2024 15:54:13 -0600 Subject: [PATCH 28/34] More Quick Start updates - Update SCM run commands, including some examples - Remove the deprecated "multirun" argument - Update argument descriptions - Update description of Docker software stack --- scm/doc/TechGuide/chap_quick.rst | 93 ++++++++++++++++---------------- 1 file changed, 46 insertions(+), 47 deletions(-) diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index 6b77a944..958868cf 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -455,16 +455,33 @@ namelists in ``../etc/case_config``. A default physics suite is provided as a us variable in the script and default namelists and tracer configurations are associated with each physics suite (through ``../src/suite_info.py``), so, technically, one must only specify a case to run with the SCM when running just one -integration. For running multiple integrations at once, one need only -specify one argument (``-m``) which runs through all permutations of supported -suites from ``../src/suite_info.py`` and cases from ``../src/supported_cases.py``. The run script’s options are described -below where option abbreviations are included in brackets. +integration. For example, to run the "BOMEX" case: + +.. code:: bash + + ./run_scm.py -c bomex + +For running multiple integrations at once, the run script can accept a file that contains a list of tests to run. +The file ``ccpp-scm/test/rt_test_cases.py`` contains the full list of regression test cases, so you could run that list +of tests with the following command: + + ./run_scm.py -f ../../test/rt_test_cases.py + +To see the full list of available options, use the ``--help`` flag: + +../../test/rt_test_cases.py +.. code:: bash + + ./run_scm.py --help + + +The run script’s full set of options are described below, where optional abbreviations are included in brackets. +If using the main branch, you should run the above command to ensure you have the most up-to-date list of options. - ``--case [-c]`` - - **This or the ``--multirun`` option are the minimum required arguments.** The - case should correspond to the name of a case in ``../etc/case_config`` (without the - ``.nml`` extension). + - **This is the only required argument.** The provided argument should correspond to the name of a case in + ``../etc/case_config`` (without the ``.nml`` extension). - ``--suite [-s]`` @@ -483,18 +500,9 @@ below where option abbreviations are included in brackets. the ``.txt`` extension). If this argument is omitted, the default tracer configuration for the given suite in ``../src/suite_info.py`` will be used. -- ``--multirun [-m]`` - - - **This or the ``--case`` option are the minimum required arguments.** When - used alone, this option runs through all permutations of supported - suites from ``../src/suite_info.py`` and cases from ``../src/supported_cases.py``. When used in conjunction with the - ``--file`` option, only the runs configured in the file will be run. - - ``--file [-f]`` - - This option may be used in conjunction with the ``--multirun`` argument. It - specifies a path and filename to a python file where multiple runs - are configured. + - This option may be used to specify a list of tests to run; see ../../test/rt_test_cases.py for an example. - ``--gdb [-g]`` @@ -505,7 +513,7 @@ below where option abbreviations are included in brackets. - Use this argument when running in a docker container in order to successfully mount a volume between the host machine and the - Docker container instance and to share the output and plots with + Docker container instance, allowing the container to share the output and plots with the host machine. - ``--runtime`` @@ -518,7 +526,7 @@ below where option abbreviations are included in brackets. - Use this to override the runtime provided in the case configuration namelist by multiplying the runtime by the given value. This is used, for example, in regression testing to reduce - total runtimes. + total runtimes (e.g., ``--runtime_mult 0.1``). - ``--levels [-l]`` @@ -574,7 +582,7 @@ configuration files located in ``../etc/case_config`` (*without the .nml extensi specifying a suite other than the default, the suite name used must match the value of the suite name in one of the suite definition files located in ``../../ccpp/suites`` (Note: not the filename of the suite definition file). As -part of the sixth CCPP release, the following suite names are valid: +part of the sixth CCPP release, the following suite names are supported: #. SCM_GFS_v16 @@ -735,32 +743,28 @@ Building the Docker image ^^^^^^^^^^^^^^^^^^^^^^^^^ The Dockerfile builds CCPP SCM v6.0.0 from source using the GNU -compiler. A number of required codes are built and installed via the -DTC-supported common community container. For reference, the common -community container repository can be accessed here: -https://github.com/NCAR/Common-Community-Container. +compiler. The CCPP SCM has a number of system requirements and necessary libraries and tools. Below is a list, including versions, used to create the the -GNU-based Docker image: +GNU-based Docker image. These are included for reference, but recall that +the Docker container contains all of this software built-in, you do not need to install them separately! -- gfortran - 9.3 +- gfortran - 12.2.0 -- gcc - 9.3 +- gcc - 12.2.0 -- cmake - 3.16.5 +- cmake - 3.25.1 -- NetCDF - 4.6.2 +- NetCDF - 4.9.0 -- HDF5 - 1.10.4 +- Python - 3.11.2 -- ZLIB - 1.2.7 +- NCEPLIBS BACIO - v2.4.1 -- SZIP - 2.1.1 +- NCEPLIBS SP - v2.3.3 -- Python - 3 - -- NCEPLIBS subset: bacio v2.4.1_4, sp v2.3.3_d, w3emc v2.9.2_d +- NCEPLIBS W3EMC - v2.11.0 A Docker image containing the SCM, CCPP, and its software prerequisites can be generated from the code in the software repository obtained by @@ -783,13 +787,15 @@ and then executing the following steps: Inspect the Dockerfile if you would like to see details for how the image is built. The image will contain SCM prerequisite software from DTC, the SCM and CCPP code, and a pre-compiled executable for the SCM - with the 6 supported suites for the SCM. A successful build will show - two images: dtcenter/common-community-container, and ccpp-scm. To - list images, type: + with the 6 supported suites for the SCM. To view .. code:: bash - docker images + > docker images + + REPOSITORY TAG IMAGE ID CREATED SIZE + ccpp-scm latest 1b2e0a0afdf9 2 days ago 3.21GB + Using a prebuilt Docker image from Dockerhub ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -838,14 +844,7 @@ Running the Docker image does not) and make sure that the “auto-mount" and “permanent" options are checked. -#. Set an environment variable to use for your SCM output directory. For - *t/csh* shells, - - .. code:: bash - - setenv OUT_DIR /path/to/output - - For bourne/bash shells, +#. Set an environment variable to use for your SCM output directory. .. code:: bash From 6cb6db864990cf3bcdd62c423fb256f1fe84bb34 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 16:00:02 -0600 Subject: [PATCH 29/34] Fix format error, convert some important notes to warnings --- scm/doc/TechGuide/chap_quick.rst | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index 958868cf..cbaf53ac 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -401,7 +401,7 @@ directory) pwd #confirm that you are in the ccpp-scm/scm/bin directory before deleting files rm -rfd * -.. note:: +.. warning:: This command can be dangerous (deletes files without confirming), so make sure that you’re in the right directory before executing! @@ -465,11 +465,12 @@ For running multiple integrations at once, the run script can accept a file that The file ``ccpp-scm/test/rt_test_cases.py`` contains the full list of regression test cases, so you could run that list of tests with the following command: +.. code:: bash + ./run_scm.py -f ../../test/rt_test_cases.py To see the full list of available options, use the ``--help`` flag: -../../test/rt_test_cases.py .. code:: bash ./run_scm.py --help @@ -909,7 +910,7 @@ Running the Docker image (remembering to include the option on run scripts if output is to be shared with the host machine). - .. note:: + .. warning:: If developing or modifying code, since the container is ephemeral, one should push their changes to a remote git repository to save them (i.e. a fork on GitHub.com). From 1d2f9e705c69d87370c58299b9060a374ed24e39 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 16:20:06 -0600 Subject: [PATCH 30/34] Fix incorrect Dockerhub org name --- .github/workflows/build_and_push_docker_latest.yml | 2 +- .github/workflows/ci_test_docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_and_push_docker_latest.yml b/.github/workflows/build_and_push_docker_latest.yml index d9dd7319..487933e3 100644 --- a/.github/workflows/build_and_push_docker_latest.yml +++ b/.github/workflows/build_and_push_docker_latest.yml @@ -7,7 +7,7 @@ on: - "main" env: - LATEST_TAG: NCAR/ccpp-scm:latest + LATEST_TAG: dtcenter/ccpp-scm:latest jobs: docker: diff --git a/.github/workflows/ci_test_docker.yml b/.github/workflows/ci_test_docker.yml index 1fac0d69..a5d36a6f 100644 --- a/.github/workflows/ci_test_docker.yml +++ b/.github/workflows/ci_test_docker.yml @@ -3,7 +3,7 @@ name: build_test_and_push_docker on: [pull_request,workflow_dispatch] env: - TEST_TAG: NCAR/ccpp-scm:test + TEST_TAG: dtcenter/ccpp-scm:test jobs: docker: From 074fbd867be70ce28f1fc49554c96ae4fb25374b Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 16:30:31 -0600 Subject: [PATCH 31/34] Apparently GitHub secrets are not passed when PRs come in from forks. This job shouldn't need those credentials anyway now that I separated out the push step, so removing the Login step should fix this --- .github/workflows/ci_test_docker.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/ci_test_docker.yml b/.github/workflows/ci_test_docker.yml index a5d36a6f..3406d832 100644 --- a/.github/workflows/ci_test_docker.yml +++ b/.github/workflows/ci_test_docker.yml @@ -15,11 +15,6 @@ jobs: uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and export test tag uses: docker/build-push-action@v5 with: From 225706201df3a627e446d467311967739ee8d3c6 Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 1 May 2024 16:35:40 -0600 Subject: [PATCH 32/34] Fix dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6212a051..7d058609 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -28,7 +28,7 @@ RUN groupadd comusers -g 9999 \ && useradd -u 9999 -g comusers -M -s /bin/bash -c "Unpriviledged user account" -d /home comuser \ && mkdir /comsoftware \ && chown -R comuser:comusers /home \ - && chmod 6755 /home + && chmod 6755 /home \ && chown -R comuser:comusers /comsoftware \ && chmod -R 6755 /comsoftware From 0283c188d8dcd4d21a794c5da8f007884e57abfe Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 15 May 2024 11:46:05 -0600 Subject: [PATCH 33/34] Updates from Dustin's comments --- scm/doc/TechGuide/chap_quick.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scm/doc/TechGuide/chap_quick.rst b/scm/doc/TechGuide/chap_quick.rst index cbaf53ac..9f407dbb 100644 --- a/scm/doc/TechGuide/chap_quick.rst +++ b/scm/doc/TechGuide/chap_quick.rst @@ -76,7 +76,7 @@ developers. While the SCM is updated with the latest commits to the CCPP submodules (ccpp-physics and ccpp-framework) -on a fairly regular basis, it may be weeks or even months behind at times. You can try to use the latest commits of the ccpp-physics and +on a fairly regular basis, it may be behind by a few commits at times. You can try to use the latest commits of the ccpp-physics and ccpp-framework submodules if you wish, but this may not have been tested (i.e. SCM development may lag ccpp-physics and/or ccpp-framework development). To do so: @@ -441,6 +441,12 @@ contents in the directory ``scm/data``. Similarly, do the same for ``thompson_tables.tar.gz`` and ``MG_INCCN_data.tar.gz`` and extract to ``scm/data/physics_input_data/``. +New with the SCM v7 release, static data is available for running cases with GOCART climatological aerosols (where the value of ``iaer`` in the ``&gfs_physics_nml`` namelist starts with 1; see the `CCPP Scientific Documentation `__ for more information); one example of this is with the default namelist settings for the GFS_v17_HR3 scheme. This dataset is very large (~12 GB), so it is recommended only to download it if you will be using it. + +.. code:: bash + + ./contrib/get_aerosol_climo.sh + .. _`singlerunscript`: Run Script Usage From 10c8f0578cab49d1580b3dd5e3012255c953e92b Mon Sep 17 00:00:00 2001 From: "Michael Kavulich, Jr" Date: Wed, 22 May 2024 10:38:29 -0600 Subject: [PATCH 34/34] Run build and push docker test on ubuntu-22.04 per concerns about "latest" tag --- .github/workflows/build_and_push_docker_latest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_push_docker_latest.yml b/.github/workflows/build_and_push_docker_latest.yml index 487933e3..4f04b7ec 100644 --- a/.github/workflows/build_and_push_docker_latest.yml +++ b/.github/workflows/build_and_push_docker_latest.yml @@ -11,7 +11,7 @@ env: jobs: docker: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - name: Checkout uses: actions/checkout@v4