diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000000..d0c3cbf102 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/make.bat b/docs/make.bat new file mode 100644 index 0000000000..6247f7e231 --- /dev/null +++ b/docs/make.bat @@ -0,0 +1,35 @@ +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=source +set BUILDDIR=build + +if "%1" == "" goto help + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.http://sphinx-doc.org/ + exit /b 1 +) + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000000..9c7258463b --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinxcontrib-bibtex +sphinx_rtd_theme diff --git a/docs/source/_static/GFS_v16_flowchart.png b/docs/source/_static/GFS_v16_flowchart.png new file mode 100644 index 0000000000..963c915768 Binary files /dev/null and b/docs/source/_static/GFS_v16_flowchart.png differ diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css new file mode 100644 index 0000000000..85a59fca39 --- /dev/null +++ b/docs/source/_static/custom.css @@ -0,0 +1,19 @@ +@import "default.css"; + +div.admonition-todo { +border-top: 2px solid red; +border-bottom: 2px solid red; +border-left: 2px solid red; +border-right: 2px solid red; +background-color: #ff6347 +} + +p.admonition-title { + display: offline; +} + +/*p.first.admonition-title { +background-color: #aa6347; +width: 100%; +} +*/ diff --git a/docs/source/_static/fv3_rocoto_view.png b/docs/source/_static/fv3_rocoto_view.png new file mode 100644 index 0000000000..02265122fe Binary files /dev/null and b/docs/source/_static/fv3_rocoto_view.png differ diff --git a/docs/source/_static/theme_overrides.css b/docs/source/_static/theme_overrides.css new file mode 100644 index 0000000000..9713e89ab2 --- /dev/null +++ b/docs/source/_static/theme_overrides.css @@ -0,0 +1,9 @@ +/* !important prevents the common CSS stylesheets from overriding this CSS since on RTD they are loaded after this stylesheet */ + +.wy-nav-content { + max-width: 100% !important; +} + +.wy-table-responsive table td { + white-space: normal !important; +} diff --git a/docs/source/clone.rst b/docs/source/clone.rst new file mode 100644 index 0000000000..e261cea00d --- /dev/null +++ b/docs/source/clone.rst @@ -0,0 +1,119 @@ +=============================== +Clone and build Global Workflow +=============================== + +^^^^^^^^^^^^^^^^^^ +Quick Instructions +^^^^^^^^^^^^^^^^^^ + +Quick clone/build/link instructions (more detailed instructions below). Note, Here we are making the assumption that you are using the workflow to run an experiment and so are working from the authoritative repository. If you are going to be a developer then follow the instructions in :doc: `development.rst`. Once you do that you can follow the instructions here with the only thing different will be the repository you are cloning from. + +For forecast-only (coupled or uncoupled):: + + git clone https://github.com/NOAA-EMC/global-workflow.git + cd global-workflow/sorc + ./checkout.sh + ./build_all.sh + ./link_workflow.sh + +For cycled (GSI):: + + git clone https://github.com/NOAA-EMC/global-workflow.git + cd global-workflow/sorc + ./checkout.sh -g + ./build_all.sh + ./link_workflow.sh + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Clone workflow and component repositories +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +******** +Workflow +******** + +https method:: + + git clone https://github.com/NOAA-EMC/global-workflow.git + +ssh method (using a password protected SSH key):: + + git clone git@github.com:NOAA-EMC/global-workflow.git + +Note: when using ssh methods you need to make sure that your GitHub account is configured for the computer from which you are accessing the repository (See `this link `_) + +Check what you just cloned (by default you will have only the develop branch):: + + cd global-workflow + git branch + * develop + +You now have a cloned copy of the global-workflow git repository. To checkout a branch or tag in your clone:: + + git checkout BRANCH_NAME + +Note: Branch must already exist. If it does not you need to make a new branch using the ``-b`` flag:: + + git checkout -b BRANCH_NAME + +The ``checkout`` command will checkout BRANCH_NAME and switch your clone to that branch. Example:: + + git checkout my_branch + git branch + * my_branch + develop + +********** +Components +********** + +Once you have cloned the workflow repository it's time to checkout/clone its components. The components will be checked out under the /sorc folder via a script called checkout.sh. Run the script with no arguments for forecast-only:: + + cd sorc + ./checkout.sh + +Or with the `-g` switch to include GSI for cycling:: + + cd sorc + ./checkout.sh -g + +If wishing to run with the operational GTG UPP and WAFS (only for select users) provide the -o flag with checkout.sh:: + + ./checkout.sh -o + +Each component cloned via checkout.sh will have a log (checkout-COMPONENT.log). Check the screen output and logs for clone errors. + +^^^^^^^^^^^^^^^^ +Build components +^^^^^^^^^^^^^^^^ + +Under the /sorc folder is a script to build all components called ``build_all.sh``. After running checkout.sh run this script to build all components codes:: + + ./build_all.sh [-a UFS_app][-c build_config][-h][-v] + -a UFS_app: + Build a specific UFS app instead of the default + -c build_config: + Selectively build based on the provided config instead of the default config + -h: + Print usage message and exit + -v: + Run all scripts in verbose mode + +A partial build option is also available via two methods: + + a) modify gfs_build.cfg config file to disable/enable particular builds and then rerun build_all.sh + + b) run individual build scripts also available in /sorc folder for each component or group of codes + +^^^^^^^^^^^^^^^ +Link components +^^^^^^^^^^^^^^^ + +At runtime the global-workflow needs all pieces in place within the main superstructure. To establish this a link script is run to create symlinks from the top level folders down to component files checked out in /sorc folders. + +After running the checkout and build scripts run the link script:: + + ./link_workflow.sh [-o] + where: + -o: Run in operations (NCO) mode. This creates copies instead of using symlinks and is generally only used by NCO during installation into production. + diff --git a/docs/source/components.rst b/docs/source/components.rst new file mode 100644 index 0000000000..0f392fc9b5 --- /dev/null +++ b/docs/source/components.rst @@ -0,0 +1,102 @@ +########################### +Global Workflow Components +########################### + +The global-workflow is a combination of several components working together to prepare, analyze, produce, and post-process forecast data. + +The major components of the system are: + +* Workflow +* Pre-processing +* Analysis +* Forecast +* Post-processing +* Verification + +The Global Workflow repository contains the workflow layer and, after running the checkout script, the code and scripts for the analysis, forecast, and post-processing components. Any non-workflow component is known as a sub-module. All of the sub-modules of the system reside in their respective repositories on GitHub. The global-workflow sub-modules are obtained by running the checkout script found under the /sorc folder. + +====================== +Component repositories +====================== + +Components checked out via sorc/checkout.sh: + +* **UFS-Weather-Model** (https://github.com/ufs-community/ufs-weather-model): This is the core model used by the Global-Workflow to provide forecasts. The UFS-weather-model repository is an umbrella repository consisting of cooupled component earth systeme that are all checked out when we check out the code at the top level of the repoitory +* **GSI** (https://github.com/NOAA-EMC/GSI): This is the core code base for atmospheric Data Assimilation +* **GSI UTILS** (https://github.com/NOAA-EMC/GSI-UTILS): Utility codes needed by GSI to create analysis +* **GSI Monitor** (https://github.com/NOAA-EMC/GSI-Monitor): These tools monitor the GSI package's data assimilation, detecting and reporting missing data sources, low observation counts, and high penalty values +* **GLDAS** (https://github.com/NOAA-EMC/GLDAS): Code base for Land Data Assimiation +* **GDAS** (https://github.com/NOAA-EMC/GDASApp): Jedi based Data Assimilation system. This system is currently being developed for marine Data Assimilation and in time will replace GSI for atmospheric data assimilation as well +* **UFS UTILS** (https://github.com/ufs-community/UFS_UTILS): Utility codes needed for UFS-weather-model +* **GFS UTILS** (https://github.com/ufs-community/gfs_utils): Utility codes needed by Global Workflow to run the GFS configuration +* **Verif global** (https://github.com/NOAA-EMC/EMC_verif-global): Verification package to evaluate GFS parallels. It uses MET and METplus. At this moment the verification package is limited to providing atmospheric metrics only +* **GFS WAFS** (https://github.com/NOAA-EMC/EMC_gfs_wafs): Additional post processing products for Aircrafts + +Note, when running the system in forecast mode only the Data Assimilation conmponents are not needed and are hence not checked out. + +===================== +External dependencies +===================== + +^^^^^^^^^ +Libraries +^^^^^^^^^ + +All the libraries that are needed to run the end to end Global Workflow are built using a package manager. Currently these are served via HPC-STACK but will soon be available via SPACK-STACK. These libraries are already available on supported NOAA HPC platforms + +Find information on official installations of HPC-STACK here: + +https://github.com/NOAA-EMC/hpc-stack/wiki/Official-Installations + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Observation data (OBSPROC/prep) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +**** +Data +**** + +Observation data, also known as dump data, is prepared in production and then archived in a global dump archive (GDA) for use by users when running cycled experiment. The GDA (identified as ``$DMPDIR`` in the workflow) is available on supported platforms and the workflow system knows where to find the data. + +* Hera: /scratch1/NCEPDEV/global/glopara/dump +* Orion: /work/noaa/rstprod/dump +* WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/dump +* S4: /data/prod/glopara/dump + +----------------------------- +Global Dump Archive Structure +----------------------------- + +The global dump archive (GDA) mimics the structure of its production source: ``DMPDIR/CDUMP.PDY/[CC/atmos/]FILES`` + +The ``CDUMP`` is either gdas, gfs, or rtofs. All three contain production output for each day (``PDY``). The gdas and gfs folders are further broken into cycle (``CC``) and component (atmos). + +The GDA also contains special versions of some datasets and experimental data that is being evaluated ahead of implementation into production. The following subfolder suffixes exist: + ++--------+------------------------------------------------------------------------------------------------------+ +| Suffix | What | ++--------+------------------------------------------------------------------------------------------------------+ +| nr | Non-restricted versions of restricted files in production. | ++--------+------------------------------------------------------------------------------------------------------+ +| ur | Un-restricted versions of restricted files in production. Produced and archived on a 48hrs delay. | ++--------+------------------------------------------------------------------------------------------------------+ +| x | Experimental global datasets being evaluated for production. Dates and types vary depending on | +| | upcoming global upgrades. | ++--------+------------------------------------------------------------------------------------------------------+ +| y | Similar to "x" but only used when there is a duplicate experimental file that is in the x subfolder | +| | with the same name. These files will be different from both the production versions | +| | (if that exists already) and the x versions. This suffix is rarely used. | ++--------+------------------------------------------------------------------------------------------------------+ +| p | Pre-production copy of full dump dataset, as produced by NCO during final 30-day parallel ahead of | +| | implementation. Not always archived. | ++--------+------------------------------------------------------------------------------------------------------+ + +*************** +Data processing +*************** + +Upstream of the global-workflow is the collection, quality control, and packaging of observed weather. The handling of that data is done by the OBSPROC group codes and scripts. The global-workflow uses two packages from OBSPROC to run its prep step to prepare observation data for use by the analysis system: + +1. https://github.com/NOAA-EMC/obsproc +2. https://github.com/NOAA-EMC/prepobs + +Both package versions and locations on support platforms are set in the global-workflow system configs. diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000000..c0f9ca572a --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,111 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('.')) + + +# -- Project information ----------------------------------------------------- + +project = 'Global-workflow' +copyright = '2023, Kate Friedman, Walter Kolczynski, Rahul Mahajan, Lin Gan, Arun Chawla' +author = 'Kate Friedman, Walter Kolczynski, Rahul Mahajan, Lin Gan, Arun Chawla' + +# The full version, including alpha/beta/rc tags +release = '0.1' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.todo', + 'sphinx.ext.coverage', + 'sphinx.ext.mathjax', + 'sphinx.ext.ifconfig', + 'sphinx.ext.viewcode', + 'sphinx.ext.githubpages', + 'sphinx.ext.napoleon', + 'sphinxcontrib.bibtex' +] + +bibtex_bibfiles = ['references.bib'] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' +html_theme_path = ["_themes", ] + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} +html_theme_options = {"body_max_width": "none"} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] +html_context = {} + + +def setup(app): + app.add_css_file('custom.css') # may also be an URL + app.add_css_file('theme_overrides.css') # may also be an URL + + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'Global-Workflow' diff --git a/docs/source/configure.rst b/docs/source/configure.rst new file mode 100644 index 0000000000..5fafddeeee --- /dev/null +++ b/docs/source/configure.rst @@ -0,0 +1,71 @@ +============= +Configure Run +============= + +The global-workflow configs contain default settings that match operations. Users wishing to run with different settings should adjust their $EXPDIR configs and then rerun the ``setup_workflow[_fcstonly].py`` script since some configuration settings/switches change the workflow/xml ("Adjusts XML" column value is "YES"). + ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| Switch | What | Default | Adjusts XML | More Details | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| EXP_WARM_START | Is experiment starting warm | .false. | NO | Impacts IAU settings for initial cycle. Can also | +| | (.true.) or cold (.false)? | | | be set when running setup_expy[_fcstonly].py | +| | | | | script with --start flag (e.g. --start warm) | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| RETRO | Use retrospective parallel | NO | NO | Default of NO will tell getic job to pull from | +| | for ICs | | | production tapes. | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| RUN_CCPP | Run CCPP | NO | NO | If YES, set desired suite for CCPP_SUITE | +| | | | | variable [#]_ | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| QUILTING | Use I/O quilting | .true. | NO | If .true. choose OUTPUT_GRID as cubed_sphere_grid | +| | | | | in netcdf or gaussian_grid | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| OUTPUT_GRID | Output grid type | gaussian_grid | NO | If gaussian_grid, set OUTPUT_FILE for nemsio or | +| | | | | netcdf | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| OUTPUT_FILE | Output file type | netcdf | NO | GFSv16 uses netcdf, GFSv15 used nemsio, recommend | +| | | | | netcdf | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| WRITE_DOPOST | Run inline post | .true. | NO | If .true. produces master post output in forecast | +| | | | | job | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| DOIAU | Enable 4DIAU for control | YES | NO | New for GFSv16 | +| | with 3 increments | | | | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| DOHYBVAR | Run EnKF | YES | YES | Don't recommend turning off | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| DONST | Run NSST | YES | NO | If YES, turns on NSST in anal/fcst steps, and | +| | | | | turn off rtgsst | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| DO_GLDAS | Run GLDAS to spin up land | YES | YES | Spins up for 84hrs if sflux files not available | +| | ICs | | | | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| DO_WAVE | Runs wave jobs and produce | YES | YES | One-way wave coupling, new for GFSv16 | +| | wave forecast output | | | | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| DO_BUFRSND | Run job to produce BUFR | NO | YES | downstream processing | +| | sounding products | | | | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| DO_GEMPAK | Run job to produce GEMPAK | NO | YES | downstream processing, ops only | +| | products | | | | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| DO_AWIPS | Run jobs to produce AWIPS | NO | YES | downstream processing, ops only | +| | products | | | | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| WAFSF | Run jobs to produce WAFS | NO | YES | downstream processing, ops only | +| | products | | | | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| DO_VRFY | Run vrfy job | NO | YES | Whether to include vrfy job (GSI monitoring, | +| | | | | tracker, VSDB, fit2obs) | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| DO_METP | Run METplus jobs | YES | YES | One cycle spinup | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| DO_VSDB | Run VSDB package | YES | NO | Retiring in GFSv17+ | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ +| LOCALARCH | Archive to a local directory | NO | Possibly | Instead of archiving data to HPSS, archive to a | +| | | | | local directory, specified by ATARDIR. If | +| | | | | LOCALARCH=YES, then HPSSARCH must =NO. Changing | +| | | | | HPSSARCH from YES to NO will adjust the XML. | ++----------------+------------------------------+---------------+-------------+---------------------------------------------------+ + +.. [#] CCPP will become default soon, variable may be removed. diff --git a/docs/source/development.rst b/docs/source/development.rst new file mode 100644 index 0000000000..6c2c0ce4fb --- /dev/null +++ b/docs/source/development.rst @@ -0,0 +1,182 @@ +################################### +Contributing to the Global Workflow +################################### + +This section is devoted to developers who wish to contribute to the Global Workflow repository. + +.. _managers: + +============= +Code managers +============= + + * Kate Friedman - @KateFriedman-NOAA / kate.friedman@noaa.gov + * Walter Kolczynski - @WalterKolczynski-NOAA / walter.kolczynski@noaa.gov + +.. _development: + +======================== +Where to do development? +======================== + + * In authoritative (main) repository: + + - Work for the upcoming implementation (who: members of global-workflow-developers team) + - Major new features or port work (who: generally code managers and/or members of global-workflow-developers team) + + * In a fork: + + - Everything and everyone else + - How do I fork this repository? See the following GitHub documentation on forking repos: https://help.github.com/en/github/getting-started-with-github/fork-a-repo + +.. _protected: + +================== +Protected branches +================== + +The following global-workflow branches are protected by the code management team: + +* develop (HEAD) +* operations (kept aligned with current production) + +These protected branches require the following to accept changes: + + 1. a pull request with at least 1 reviewer sign-off + 2. a code manager to perform the commit + +Other authoritative repository branches may also be protected at the request of members of the global-workflow-developers team. + +.. _howto: + +============================================= +How to get changes into develop (HEAD) branch +============================================= + +The following steps should be followed in order to make changes to the develop branch of global-workflow. Communication with the code managers throughout the process is encouraged. + + #. Issue - Open issue to document changes. Reference this issue in commits to your branches (e.g. ``git commit -m "Issue #23 - blah changes for what-not code"``) Click `here `__ to open a new global-workflow issue. + #. GitFlow - Follow `GitFlow `_ procedures for development (branch names, forking vs branching, etc.). Read more `here `__ about GitFlow at EMC. + #. To fork or not to fork? - If not working within authoritative repository create a fork of the authoritative repository. Read more `here `__ about forking in GitHub. + #. Branch - Create branch in either authoritative repository or fork of authoritative repository. See the `Where to do development? `_ section for how to determine where. Follow Gitflow conventions when creating branch. + #. Development - Perform and test changes in branch. Document work in issue and mention issue number in commit messages to link your work to the issue See `Commit Messages `_ section below. Depending on changes the code manager may request or perform additional pre-commit tests. + #. Pull request - When ready to merge changes back to develop branch, the lead developer should initiate a pull request (PR) of your branch (either fork or not) into the develop branch. Read `here `__ about pull requests in GitHub. Provide some information about the PR in the proper field, add at least one reviewer to the PR and assign the PR to a code manager. + #. Complete - When review and testing is complete the code manager will complete the pull request and subsequent merge/commit. + #. Cleanup - When complete the lead developer should delete the branch and close the issue. "Closing keywords" can be used in the PR to automatically close associated issues. + +.. _code-standards: + +============== +Code standards +============== + +All scripts should be in either bash or python 3. + +We have adopted the `Google style guide `_ for shell scripts and `PEP-8 `_ for python. Python code should additionally have docstrings following `numpy style `_. + +All new code after 2022 Sep 1 will be required to meet these standards. We will slowly be updating existing scripts to comply with the standards. We are also in the process of adding GitHub actions to automatically lint code submitted for PRs. + +.. _commit-standards: + +======================== +Commit message standards +======================== + +**ALL** commits must follow best practices for commit messages: https://chris.beams.io/posts/git-commit/ + + * Separate subject from body with a blank line + * Limit the subject line to 50 characters + * Capitalize the subject line + * Do not end the subject line with a period + * Use the `imperative mood `_ in the subject line + * Wrap the body at 72 characters + * Use the body to explain what and why vs. how + * The final line of the commit message should include tags to relevant issues (e.g. ``Refs: #217, #300``) + +Here is the example commit message from the article linked above; it includes descriptions of what would be in each part of the commit message for guidance: + + Summarize changes in around 50 characters or less + + More detailed explanatory text, if necessary. Wrap it to about 72 + characters or so. In some contexts, the first line is treated as the + subject of the commit and the rest of the text as the body. The + blank line separating the summary from the body is critical (unless + you omit the body entirely); various tools like `log`, `shortlog` + and `rebase` can get confused if you run the two together. + + Explain the problem that this commit is solving. Focus on why you + are making this change as opposed to how (the code explains that). + Are there side effects or other unintuitive consequences of this + change? Here's the place to explain them. + + Further paragraphs come after blank lines. + + - Bullet points are okay, too + + - Typically a hyphen or asterisk is used for the bullet, preceded + by a single space, with blank lines in between, but conventions + vary here + + If you use an issue tracker, put references to them at the bottom, + like this:: + + Resolves: #123 + See also: #456, #789 + +A detailed commit message is very useful for documenting changes + +.. _sync: + +================================================== +How to sync fork with the authoritative repository +================================================== + +As development in the main authoritative repository moves forward you will need to sync your fork's branches to stay up-to-date. Below is an example of how to sync your fork's copy of a branch with the authoritative repository copy. The branch name for the example will be "feature/new_thing". Click `here `__ for documentation on syncing forks. + +1. Clone your fork and checkout branch that needs syncing + +:: + + git clone https://github.com/JoeSchmo-NOAA/global-workflow.git ./fork + cd fork + git checkout feature/my_new_thing + +2. Add upstream info to your clone so it knows where to merge from. The term "upstream" refers to the authoritative repository from which the fork was create + +:: + + git remote add upstream https://github.com/NOAA-EMC/global-workflow.git + +3. Fetch upstream information into clone + +:: + + git fetch upstream + +Later on you can update your fork's remote information by doing the following command + +:: + + git remote update + +4. Merge upstream feature/other_new_thing into your branch + +:: + + git merge upstream/feature/other_new_thing + +5. Resolve any conflicts and perform any needed "add"s or "commit"s for conflict resolution. + +6. Push the merged copy back up to your fork (origin) + +:: + + git push origin feature/my_new_thing + +7. Note the sync in the GitHub issue tracking your work. + +Done! + +Moving forward you'll want to perform the "remote update" command regularly to update the metadata for the remote/upstream repository in your fork (e.g. pull in metadata for branches made in auth repo after you forked it):: + + git remote update diff --git a/docs/source/errors_faq.rst b/docs/source/errors_faq.rst new file mode 100644 index 0000000000..82830d5110 --- /dev/null +++ b/docs/source/errors_faq.rst @@ -0,0 +1,48 @@ +========================== +Common Errors Known Issues +========================== + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Error: "ImportError" message when running setup script +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + $ ./setup_xml.py /path/to/your/experiment/directory + Traceback (most recent call last): + File "./setup_workflow.py", line 32, in + from collections import OrderedDict + ImportError: cannot import name OrderedDict + +**Cause:** Missing python module in your environment + +**Solution:** Load a python module ("module load python") and retry setup script. + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Error: curses default colors when running viewer +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Example:: + + $ ./rocoto_viewer.py -d blah.db -w blah.xml + Traceback (most recent call last): + File "./rocoto_viewer.py", line 2376, in + curses.wrapper(main) + File "/contrib/anaconda/anaconda2/4.4.0/lib/python2.7/curses/wrapper.py", line 43, in wrapper + return func(stdscr, *args, **kwds) + File "./rocoto_viewer.py", line 1202, in main + curses.use_default_colors() + _curses.error: use_default_colors() returned ERR + +**Cause:** wrong TERM setting for curses + +**Solution:** set TERM to "xterm" (bash: export TERM=xterm ; csh/tcsh: setenv TERM xterm) + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Issue: Directory name change for EnKF folder in COMROT +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Issue:** The EnKF COMROT folders were renamed during the GFS v15 development process to remove the period between "enkf" and "gdas": enkf.gdas.$PDY → enkfgdas.$PDY + +**Fix:** Older tarballs on HPSS will have the older directory name with the period between 'enkf' and 'gdas'. Make sure to rename folder to 'enkfgdas.$PDY' after obtaining. Only an issue for the initial cycle. + diff --git a/docs/source/hpc.rst b/docs/source/hpc.rst new file mode 100644 index 0000000000..a92bc79e1a --- /dev/null +++ b/docs/source/hpc.rst @@ -0,0 +1,120 @@ +##################### +HPC Settings and Help +##################### + +Running the GFS configurations (or almost any global workflow configuration except the coarsest) is a resource intensive exercise. This page discusses recommended HPC environmental settings and contact information in case you need assistance from a particular HPC helpdesk. While most of the documentation is based on supported NOAA platforms, the learnings here can hopefully apply to other platforms. + +================================ +Experiment troubleshooting help +================================ + +Users may email Kate Friedman (kate.friedman@noaa.gov) questions or requests for troubleshooting assistance with their global-workflow experiments/parallels on supported platforms. For troubleshooting, please provide a brief description of the issue(s) and include relevant error messages and/or paths to logs for failed jobs. + +Any issues related to HPC/machine problems, and which are unrelated to the workflow itself, should go to the appropriate HPC helpdesk. + +============= +HPC helpdesks +============= + +* WCOSS2: hpc.wcoss2-help@noaa.gov +* Hera: rdhpcs.hera.help@noaa.gov +* Orion: rdhpcs.orion.help@noaa.gov +* HPSS: rdhpcs.hpss.help@noaa.gov +* Gaea: oar.gfdl.help@noaa.gov +* S4: david.huber@noaa.gov + +====================== +Restricted data access +====================== + +The GFS system ingests dump data files that contain global observation data. A number of these dump files contain restricted data which means those files come with an extra level of permissions called restricted or ‘rstprod’. Users who wish to run cycled GFS experiments, which both utilizes restricted observation data and produces output containing restricted data, will need to gain rstprod group access. + +NOTE: Only non-restricted data is available on S4. + +To request rstprod access, do either a and/or b below: + +a) If you need restricted data access on WCOSS, fill out form here: + +https://www.nco.ncep.noaa.gov/sib/restricted_data/restricted_data_sib/ + +b) If you need restricted data access on RDHPCS systems: go to the AIM system, click on "Request new access to a project", select the rstprod project, provide justification for needed access, and submit the request: + +https://aim.rdhpcs.noaa.gov/ + +==================================== +Optimizing the global workflow on S4 +==================================== + +The S4 cluster is relatively small and so optimizations are recommended to improve cycled runtimes. Please contact David Huber (david.huber@noaa.gov) if you are planning on running a cycled experiment on this system to obtain optimized configuration files. + +============ +Git settings +============ + +^^^^^^ +Merges +^^^^^^ + +Use the following command to have merge commits include the one-line description of all the commits being merged (up to 200). You only need to do this once on each machine; it will be saved to your git settings:: + + git config --global merge.log 200 + +Use the ``--no-ff`` option to make sure there is always a merge commit when a fast-forward only is available. Exception: If the merge contains only a single commit, it can be applied as a fast-forward. + +For any merge with multiple commits, a short synopsis of the merge should appear between the title and the list of commit titles added by merge.log. + +^^^^^^^ +Version +^^^^^^^ + +It is advised to use Git v2+ when available. At the time of writing this documentation the swfault Git clients on the different machines were as noted in the table below. It is recommended that you check the default modules before loading recommneded ones: + ++---------+----------+---------------------------------------+ +| Machine | Default | Recommended | ++---------+----------+---------------------------------------+ +| Hera | v2.18.0 | default | ++---------+----------+---------------------------------------+ +| Orion | v1.8.3.1 | **module load git/2.28.0** | ++---------+----------+---------------------------------------+ +| WCOSS2 | v2.26.2 | default or **module load git/2.29.0** | ++---------+----------+---------------------------------------+ +| S4 | v1.8.3.1 | **module load git/2.30.0** | ++---------+----------+---------------------------------------+ + +^^^^^^^^^^^^^ +Output format +^^^^^^^^^^^^^ + +For proper display of Git command output (e.g. git branch and git diff) type the following once on both machines:: + + git config --global core.pager 'less -FRX' + +For the manage_externals utility functioning:: + + Error: fatal: ssh variant 'simple' does not support setting port + Fix: git config --global ssh.variant ssh + +=================================== +Stacksize on R&Ds (Hera, Orion, S4) +=================================== + +Some GFS components, like the UPP, need an unlimited stacksize. Add the following setting into your appropriate .*rc file to support these components: + +csh:: + + limit stacksize unlimited + +sh/bash/ksh:: + + ulimit -s unlimited + +========================================= +Forecast hangs due to issue with ssh-keys +========================================= + +Did you generate your ssh-keys with a passphrase? If so, remake them without one. To test this try ssh-ing to a different login node; you should be able to without being prompted for your passphrase. + +Is your public key in the authorized_keys file? If not, add it:: + + cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys + diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000000..7a73e88c26 --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,40 @@ + +############### +Global Workflow +############### + +**Global-workflow** is the end-to-end workflow designed to run global configurations of medium range weather forecasting for the UFS weather model. It supports both development and operational implementations. In its current format it supports the Global Forecast System (GFS) and the Gobal Ensemble Forecast System (GEFS) configurations + +====== +Status +====== + +* State of develop (HEAD) branch: early GFSv17+ development + +* State of operations branch: GFS v16.3.5 `tag: [gfs.v16.3.5] `_ + +============== +Code managers: +============== + +* Kate Friedman - @KateFriedman-NOAA / kate.friedman@noaa.gov +* Walter Kolczynski - @WalterKolczynski-NOAA / walter.kolczynski@noaa.gov + +============== +Announcements: +============== + +General updates: NOAA employees and affiliates can join the gfs-announce distribution list to get updates on the GFS and global-workflow. Contact Kate Friedman (kate.friedman@noaa.gov) and Walter Kolczynski (walter.kolczynski@noaa.gov) to get added to the list or removed from it. + +GitHub updates: Users should adjust their "Watch" settings for this repo so they receive notifications as they'd like to. Find the "Watch" or "Unwatch" button towards the top right of this page and click it to adjust how you watch this repo. + +.. toctree:: + :numbered: + :maxdepth: 3 + + development.rst + components.rst + jobs.rst + hpc.rst + output.rst + run.rst diff --git a/docs/source/init.rst b/docs/source/init.rst new file mode 100644 index 0000000000..e674260319 --- /dev/null +++ b/docs/source/init.rst @@ -0,0 +1,332 @@ +========================== +Prepare Initial Conditions +========================== + +There are two types of initial conditions for the global-workflow: + +#. Warm start: these ICs are taken directly from either the GFS in production or an experiment "warmed" up (at least one cycle in). +#. Cold start: any ICs converted to a new resolution or grid (e.g. GSM-GFS -> FV3GFS). These ICs are often prepared by chgres_cube (change resolution utility). + +Most users will initiate their experiments with cold start ICs unless running high resolution (C768 deterministic with C384 EnKF) for a date with warm starts available. It is `not recommended` to run high resolution unless required or as part of final testing. + +Resolutions: + +* C48 = 2­ degree ≈ 200km +* C96 = 1­ degree ≈ 100km +* C192 = 1/2­ degree ≈ 50km +* C384 = 1/4 degree ≈ 25km +* C768 = 1/8th degree ≈ 13km +* C1152 ≈ 9km +* C3072 ≈ 3km + +Supported resolutions in global-workflow: C48, C96, C192, C384, C768 + +^^^^^^^^^^^^^^^^^^^^ +Automated Generation +^^^^^^^^^^^^^^^^^^^^ + +*********** +Cycled mode +*********** + +Not yet supported. See Manual Generation section below for how to create your ICs yourself (outside of workflow). + +***************************** +Free-forecast mode (atm-only) +***************************** + +Free-forecast mode in global workflow includes ``getic`` and ``init`` jobs for the gfs suite. The ``getic`` job pulls inputs for ``chgres_cube`` (init job) or warm start ICs into your ``ROTDIR/COMROT``. The ``init`` job then ingests those files to produce initial conditions for your experiment. + +Users on machines without HPSS access (e.g. Orion) need to perform the ``getic`` step manually and stage inputs for the ``init`` job. The table below lists the needed files for ``init`` and where to place them in your ``ROTDIR``. + +Note for table: yyyy=year; mm=month; dd=day; hh=cycle + +Operations/production output location on HPSS: /NCEPPROD/hpssprod/runhistory/rh ``yyyy``/``yyyymm``/``yyyymmdd``/ + ++----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ +| Source | Files | Tarball name | Where in ROTDIR | ++----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ +| v12 ops | gfs.t. ``hh`` z.sanl | com_gfs_prod_gfs. ``yyyymmddhh`` .anl.tar | gfs. ``yyyymmdd`` /``hh`` | +| | | | | +| | gfs.t. ``hh`` z.sfcanl | | | ++----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ +| v13 ops | gfs.t. ``hh`` z.sanl | com2_gfs_prod_gfs. ``yyyymmddhh`` .anl.tar | gfs. ``yyyymmdd`` /``hh`` | +| | | | | +| | gfs.t. ``hh`` z.sfcanl | | | ++----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ +| v14 ops | gfs.t. ``hh`` z.atmanl.nemsio | gpfs_hps_nco_ops_com_gfs_prod_gfs. ``yyyymmddhh`` .anl.tar | gfs. ``yyyymmdd` /`hh`` | +| | | | | +| | gfs.t. ``hh`` z.sfcanl.nemsio | | | ++----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ +| v15 ops | gfs.t. ``hh`` z.atmanl.nemsio | gpfs_dell1_nco_ops_com_gfs_prod_gfs. ``yyyymmdd`` _ ``hh`` .gfs_nemsioa.tar | gfs. ``yyyymmdd`` /``hh`` | +| | | | | +| pre-2020022600 | gfs.t. ``hh`` z.sfcanl.nemsio | | | ++----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ +| v15 ops | gfs.t. ``hh`` z.atmanl.nemsio | com_gfs_prod_gfs. ``yyyymmdd`` _ ``hh`` .gfs_nemsioa.tar | gfs. ``yyyymmdd`` /``hh`` | +| | | | | +| | gfs.t. ``hh`` z.sfcanl.nemsio | | | ++----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ +| v16 ops | gfs.t. ``hh`` z.atmanl.nc | com_gfs_prod_gfs. ``yyyymmdd`` _ ``hh`` .gfs_nca.tar | gfs. ``yyyymmdd`` /``hh``/atmos| +| | | | | +| | gfs.t. ``hh`` z.sfcanl.nc | | | ++----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ +| v16 retro | gfs.t. ``hh`` z.atmanl.nc | gfs_netcdfa.tar* | gfs. ``yyyymmdd`` /``hh``/atmos| +| | | | | +| | gfs.t. ``hh`` z.sfcanl.nc | | | ++----------------+---------------------------------+-----------------------------------------------------------------------------+--------------------------------+ + +For HPSS path, see retrospective table in :ref:`pre-production parallel section `: below + +********************* +Free-forecast coupled +********************* + +Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Copies of initial conditions from the prototype runs are currently maintained on Hera, Orion, and WCOSS2. The locations used are determined by ``parm/config/config.coupled_ic``. If you need prototype ICs on another machine, please contact Walter (Walter.Kolczynski@noaa.gov). + +^^^^^^^^^^^^^^^^^ +Manual Generation +^^^^^^^^^^^^^^^^^ + +NOTE: Initial conditions cannot be generated on S4. These must be generated on another supported platform then pushed to S4. If you do not have access to a supported system or need assistance, please contact David Huber (david.huber@noaa.gov). + +*********** +Cold starts +*********** + +The following information is for users needing to generate initial conditions for a cycled experiment that will run at a different resolution or layer amount than the operational GFS (C768C384L127). + +The ``chgres_cube`` code is available from the `UFS_UTILS repository `_ on GitHub and can be used to convert GFS ICs to a different resolution or number of layers. Users may clone the develop/HEAD branch or the same version used by global-workflow develop (found in sorc/checkout.sh). The ``chgres_cube`` code/scripts currently support the following GFS inputs: + +* pre-GFSv14 +* GFSv14 +* GFSv15 +* GFSv16 + +Clone UFS_UTILS:: + + git clone --recursive https://github.com/NOAA-EMC/UFS_UTILS.git + +Then switch to a different tag or use the default branch (develop). + +Build UFS_UTILS:: + + sh build_all.sh + cd fix + sh link_fixdirs.sh emc $MACHINE + +where ``$MACHINE`` is ``wcoss2``, ``hera``, ``jet``, or ``orion``. Note: UFS-UTILS builds on Orion but due to the lack of HPSS access on Orion the ``gdas_init`` utility is not supported there. + +Configure your conversion:: + + cd util/gdas_init + vi config + +Read the doc block at the top of the config and adjust the variables to meet you needs (e.g. ``yy, mm, dd, hh`` for ``SDATE``). + +Submit conversion script::` + + ./driver.$MACHINE.sh + +where ``$MACHINE`` is currently ``wcoss2``, ``hera`` or ``jet``. Additional options will be available as support for other machines expands. Note: UFS-UTILS builds on Orion but due to lack of HPSS access there is no ``gdas_init`` driver for Orion nor support to pull initial conditions from HPSS for the ``gdas_init`` utility. + +3 small jobs will be submitted: + + - 1 jobs to pull inputs off HPSS + - 2 jobs to run ``chgres_cube`` (1 for deterministic/hires and 1 for each EnKF ensemble member) + +The chgres jobs will have a dependency on the data-pull jobs and will wait to run until all data-pull jobs have completed. + +Check output: + +In the config you will have defined an output folder called ``$OUTDIR``. The converted output will be found there, including the needed abias and radstat initial condition files. The files will be in the needed directory structure for the global-workflow system, therefore a user can move the contents of their ``$OUTDIR`` directly into their ``$ROTDIR/$COMROT``. + +Please report bugs to George Gayno (george.gayno@noaa.gov) and Kate Friedman (kate.friedman@noaa.gov). + +***************************** +Warm starts (from production) +***************************** + +The GFSv15 was implemented into production on June 12th, 2019 at 12z. The GFS was spun up ahead of that cycle and thus production output for the system is available from the 00z cycle (2019061200) and later. Production output tarballs from the prior GFSv14 system are located in the same location on HPSS but have "hps" in the name to represent that it was run on the Cray, where as the GFS now runs in production on the Dell and has "dell1" in the tarball name. + +See production output in the following location on HPSS: + +``/NCEPPROD/hpssprod/runhistory/rhYYYY/YYYYMM/YYYYMMDD`` + +Example location: + +``/NCEPPROD/hpssprod/runhistory/rh2021/202104/20210420`` + +Example listing for 2021042000 production tarballs:: + + [Kate.Friedman@m72a2 ~]$ hpsstar dir /NCEPPROD/hpssprod/runhistory/rh2021/202104/20210420 | grep gfs | grep _00. | grep -v idx + [connecting to hpsscore1.fairmont.rdhpcs.noaa.gov/1217] + ****************************************************************** + * Welcome to the NESCC High Performance Storage System * + * * + * Current HPSS version: 7.5.3 * + * * + * * + * Please Submit Helpdesk Request to * + * rdhpcs.hpss.help@noaa.gov * + * * + * Announcements: * + ****************************************************************** + Username: Kate.Friedman UID: 2391 Acct: 2391(2391) Copies: 1 COS: 0 Firewall: off [hsi.6.3.0.p1-hgs Thu May 7 09:16:23 UTC 2020] + /NCEPPROD/hpssprod/runhistory/rh2021/202104: + drwxr-xr-x 2 nwprod prod 11776 Apr 19 23:44 20210420 + [connecting to hpsscore1.fairmont.rdhpcs.noaa.gov/1217] + -rw-r----- 1 nwprod rstprod 51268255744 Apr 22 05:29 com_gfs_prod_enkfgdas.20210420_00.enkfgdas.tar + -rw-r--r-- 1 nwprod prod 220121310720 Apr 22 06:42 com_gfs_prod_enkfgdas.20210420_00.enkfgdas_restart_grp1.tar + -rw-r--r-- 1 nwprod prod 220124178944 Apr 22 07:04 com_gfs_prod_enkfgdas.20210420_00.enkfgdas_restart_grp2.tar + -rw-r--r-- 1 nwprod prod 220120305664 Apr 22 07:24 com_gfs_prod_enkfgdas.20210420_00.enkfgdas_restart_grp3.tar + -rw-r--r-- 1 nwprod prod 220116934656 Apr 22 07:38 com_gfs_prod_enkfgdas.20210420_00.enkfgdas_restart_grp4.tar + -rw-r--r-- 1 nwprod prod 220121547776 Apr 22 07:56 com_gfs_prod_enkfgdas.20210420_00.enkfgdas_restart_grp5.tar + -rw-r--r-- 1 nwprod prod 220125794816 Apr 22 08:09 com_gfs_prod_enkfgdas.20210420_00.enkfgdas_restart_grp6.tar + -rw-r--r-- 1 nwprod prod 220117037568 Apr 22 08:23 com_gfs_prod_enkfgdas.20210420_00.enkfgdas_restart_grp7.tar + -rw-r--r-- 1 nwprod prod 220117203968 Apr 22 08:33 com_gfs_prod_enkfgdas.20210420_00.enkfgdas_restart_grp8.tar + -rw-r----- 1 nwprod rstprod 9573153280 Apr 22 02:49 com_gfs_prod_gdas.20210420_00.gdas.tar + -rw-r--r-- 1 nwprod prod 1020249088 Apr 22 02:49 com_gfs_prod_gdas.20210420_00.gdas_flux.tar + -rw-r--r-- 1 nwprod prod 92950728704 Apr 22 03:05 com_gfs_prod_gdas.20210420_00.gdas_nc.tar + -rw-r--r-- 1 nwprod prod 10647806464 Apr 22 02:50 com_gfs_prod_gdas.20210420_00.gdas_pgrb2.tar + -rw-r----- 1 nwprod rstprod 65121796608 Apr 22 02:56 com_gfs_prod_gdas.20210420_00.gdas_restart.tar + -rw-r--r-- 1 nwprod prod 18200814080 Apr 22 03:06 com_gfs_prod_gdas.20210420_00.gdaswave_keep.tar + -rw-r----- 1 nwprod rstprod 13013076992 Apr 22 03:08 com_gfs_prod_gfs.20210420_00.gfs.tar + -rw-r--r-- 1 nwprod prod 62663230976 Apr 22 03:13 com_gfs_prod_gfs.20210420_00.gfs_flux.tar + -rw-r--r-- 1 nwprod prod 127932879360 Apr 22 03:47 com_gfs_prod_gfs.20210420_00.gfs_nca.tar + -rw-r--r-- 1 nwprod prod 138633526272 Apr 22 04:00 com_gfs_prod_gfs.20210420_00.gfs_ncb.tar + -rw-r--r-- 1 nwprod prod 140773240832 Apr 22 03:27 com_gfs_prod_gfs.20210420_00.gfs_pgrb2.tar + -rw-r--r-- 1 nwprod prod 61253672960 Apr 22 03:32 com_gfs_prod_gfs.20210420_00.gfs_pgrb2b.tar + -rw-r--r-- 1 nwprod prod 19702107136 Apr 22 03:34 com_gfs_prod_gfs.20210420_00.gfs_restart.tar + -rw-r--r-- 1 nwprod prod 18617610240 Apr 22 04:02 com_gfs_prod_gfs.20210420_00.gfswave_output.tar + -rw-r--r-- 1 nwprod prod 30737774592 Apr 22 04:05 com_gfs_prod_gfs.20210420_00.gfswave_raw.tar + +The warm starts and other output from production are at C768 deterministic and C384 EnKF. The warm start files must be converted to your desired resolution(s) using ``chgres_cube`` if you wish to run a different resolution. If you are running a C768/C384 experiment you can use them as is. + +.. _fix-netcdf: + +------------------------- +Fix NetCDF checksum issue +------------------------- + +Due to a recent change in UFS, the setting to bypass the data verification no longer works, so you may also need an additional offline step to delete the checksum of the NetCDF files for warm start: + +On RDHPCS:: + + module load nco/4.9.3 + +On WCOSS2:: + + module load intel/19.1.3.304 + module load netcdf/4.7.4 + module load udunits/2.2.28 + module load gsl/2.7 + module load nco/4.7.9 + +And then on all platforms:: + + cd $COMROT + for f in $(find ./ -name *tile*.nc); do echo $f; ncatted -a checksum,,d,, $f; done + +------------------------------------------------------------------------------------------ +What files should you pull for starting a new experiment with warm starts from production? +------------------------------------------------------------------------------------------ + +That depends on what mode you want to run -- free-forecast or cycled. Whichever mode navigate to the top of your ``COMROT`` and pull the entirety of the tarball(s) listed below for your mode. The files within the tarball are already in the ``$CDUMP.$PDY/$CYC`` folder format expected by the system. + +For free-forecast there are two tar balls to pull + + 1. File #1 (for starting cycle SDATE):: + /NCEPPROD/hpssprod/runhistory/rhYYYY/YYYYMM/YYYYMMDD/com_gfs_prod_gfs.YYYYMMDD_CC.gfs_restart.tar + 2. File #2 (for prior cycle GDATE=SDATE-06):: + /NCEPPROD/hpssprod/runhistory/rhYYYY/YYYYMM/YYYYMMDD/com_gfs_prod_gdas.YYYYMMDD_CC.gdas_restart.tar + + For cycled mode there 18 tarballs to pull (9 for SDATE and 9 for GDATE (SDATE-06)):: + + HPSS path: /NCEPPROD/hpssprod/runhistory/rhYYYY/YYYYMM/YYYYMMDD/ + +Tarballs per cycle:: + + com_gfs_prod_gdas.YYYYMMDD_CC.gdas_restart.tar + com_gfs_prod_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp1.tar + com_gfs_prod_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp2.tar + com_gfs_prod_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp3.tar + com_gfs_prod_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp4.tar + com_gfs_prod_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp5.tar + com_gfs_prod_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp6.tar + com_gfs_prod_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp7.tar + com_gfs_prod_enkfgdas.YYYYMMDD_CC.enkfgdas_restart_grp8.tar + +Go to the top of your ``COMROT/ROTDIR`` and pull the contents of all tarballs there. The tarballs already contain the needed directory structure. + +******************************************* +Warm starts (from pre-production parallels) +******************************************* + +Recent pre-implementation parallel series was for GFS v16 (implemented March 2021). For the prior v15 (Q2FY19) see an additional table below. + +* **What resolution are warm-starts available for?** Warm-start ICs are saved at the resolution the model was run at (C768/C384) and can only be used to run at the same resolution combination. If you need to run a different resolution you will need to make your own cold-start ICs. See cold start section above. +* **What dates have warm-start files saved?** Unfortunately the frequency changed enough during the runs that it’s not easy to provide a definitive list easily. +* **What files?** All warm-starts are saved in separate tarballs which include “restart” in the name. You need to pull the entirety of each tarball, all files included in the restart tarballs are needed. +* **Where are these tarballs?** See below for the location on HPSS for each v16 pre-implementation parallel. +* **What tarballs do I need to grab for my experiment?** Tarballs from two cycles are required. The tarballs are listed below, where $CDATE is your starting cycle and $GDATE is one cycle prior. + + - Free-forecast + + ../$CDATE/gfs_restarta.tar + + ../$GDATE/gdas_restartb.tar + - Cycled w/EnKF + + ../$CDATE/gdas_restarta.tar + + ../$CDATE/enkfgdas_restarta_grp##.tar (where ## is 01 through 08) (note, older tarballs may include a period between enkf and gdas: "enkf.gdas") + + ../$GDATE/gdas_restartb.tar + + ../$GDATE/enkfgdas_restartb_grp##.tar (where ## is 01 through 08) (note, older tarballs may include a period between enkf and gdas: "enkf.gdas") + +* **Where do I put the warm-start initial conditions?** Extraction should occur right inside your COMROT. You may need to rename the enkf folder (enkf.gdas.$PDY -> enkfgdas.$PDY). + +Due to a recent change in the dycore, you may also need an additional offline step to fix the checksum of the NetCDF files for warm start. See the :ref:`fix netcdf checksum section `: above + +.. _retrospective: + +-------------------------------------------------------------- +GFSv16 (March 2021) Pre-Implementation Parallel HPSS Locations +-------------------------------------------------------------- + ++-----------------------------+---------------+--------------------------------------------------+ +| Time Period | Parallel Name | Archive Location on HPSS | +| | | PREFIX=/NCEPDEV/emc-global/5year/emc.glopara | ++-----------------------------+---------------+--------------------------------------------------+ +| 2019050106 ~ 2019060100 | v16retro0e | $PREFIX/WCOSS_D/gfsv16/v16retro0e/``yyyymmddhh`` | ++-----------------------------+---------------+--------------------------------------------------+ +| 2019060106 ~ 2019083118 | v16retro1e | $PREFIX/WCOSS_D/gfsv16/v16retro1e/``yyyymmddhh`` | ++-----------------------------+---------------+--------------------------------------------------+ +| 2019090100 ~ 2019110918 | v16retro2e | $PREFIX/WCOSS_D/gfsv16/v16retro2e/``yyyymmddhh`` | ++-----------------------------+---------------+--------------------------------------------------+ +| 2019111000 ~ 2020122200 | v16rt2 | $PREFIX/WCOSS_D/gfsv16/v16rt2/``yyyymmddhh`` | ++-----------------------------+---------------+--------------------------------------------------+ +| 2020122206 ~ implementation | v16rt2n | $PREFIX/WCOSS_D/gfsv16/v16rt2n/``yyyymmddhh`` | ++-----------------------------+---------------+--------------------------------------------------+ + +---------------------------------------------------------- +GFSv15 (Q2FY19) Pre-Implementation Parallel HPSS Locations +---------------------------------------------------------- + ++---------------------+-----------------+-----------------------------------------------------------+ +| Time Period | Parallel Name | Archive Location on HPSS | +| | | PREFIX=/NCEPDEV/emc-global/5year | ++---------------------+-----------------+-----------------------------------------------------------+ +| 20180525 - 20190612 | prfv3rt1 | $PREFIX/emc.glopara/WCOSS_C/Q2FY19/prfv3rt1 | ++---------------------+-----------------+-----------------------------------------------------------+ +| 20171125 - 20170831 | fv3q2fy19retro1 | $PREFIX/Fanglin.Yang/WCOSS_DELL_P3/Q2FY19/fv3q2fy19retro1 | ++---------------------+-----------------+-----------------------------------------------------------+ +| 20170525 - 20170625 | fv3q2fy19retro2 | $PREFIX/emc.glopara/WCOSS_C/Q2FY19/fv3q2fy19retro2 | ++---------------------+-----------------+-----------------------------------------------------------+ +| 20170802 - 20171130 | fv3q2fy19retro2 | $PREFIX/Fanglin.Yang/WCOSS_DELL_P3/Q2FY19/fv3q2fy19retro2 | ++---------------------+-----------------+-----------------------------------------------------------+ +| 20161125 - 20170531 | fv3q2fy19retro3 | $PREFIX/Fanglin.Yang/WCOSS_DELL_P3/Q2FY19/fv3q2fy19retro3 | ++---------------------+-----------------+-----------------------------------------------------------+ +| 20160817 - 20161130 | fv3q2fy19retro4 | $PREFIX/emc.glopara/WCOSS_DELL_P3/Q2FY19/fv3q2fy19retro4 | ++---------------------+-----------------+-----------------------------------------------------------+ +| 20160522 - 20160825 | fv3q2fy19retro4 | $PREFIX/emc.glopara/WCOSS_C/Q2FY19/fv3q2fy19retro4 | ++---------------------+-----------------+-----------------------------------------------------------+ +| 20151125 - 20160531 | fv3q2fy19retro5 | $PREFIX/emc.glopara/WCOSS_DELL_P3/Q2FY19/fv3q2fy19retro5 | ++---------------------+-----------------+-----------------------------------------------------------+ +| 20150503 - 20151130 | fv3q2fy19retro6 | $PREFIX/emc.glopara/WCOSS_DELL_P3/Q2FY19/fv3q2fy19retro6 | ++---------------------+-----------------+-----------------------------------------------------------+ diff --git a/docs/source/jobs.rst b/docs/source/jobs.rst new file mode 100644 index 0000000000..249f305b07 --- /dev/null +++ b/docs/source/jobs.rst @@ -0,0 +1,89 @@ +################# +GFS Configuration +################# + +.. figure:: _static/GFS_v16_flowchart.png + + Schematic flow chart for GFS v16 in operations + +The sequence of jobs that are run for an end-to-end (DA+forecast+post processing+verification) GFS configuration using the Global Workflow is shown above. The system utilizes a collection of scripts that perform the tasks for each step. + +For any cycle the system consists of two phases -- the gdas phase which provides the initial guess fields, and the gfs phase which creates the initial conditions and forecast of the system. As with the operational system, the gdas runs for each cycle (00, 06, 12, and 18 UTC), however, to save time and space in experiments, the gfs (right side of the diagram) is initially setup to run for only the 00 UTC cycle. (See the "run GFS this cycle?" portion of the diagram) The option to run the GFS for all four cycles is available (see gfs_cyc variable in configuration file). + +An experimental run is different from operations in the following ways: + +* Workflow manager: operations utilizes `ecFlow `__, while development currently utilizes `ROCOTO `__. Note, experiments can also be run using ecFlow. + +* Dump step is not run as it has already been completed during the real-time production runs and dump data is available in the global dump archive on supported machines. + +* Addition steps in experimental mode: + + - verification (vrfy) + + - archive (arch) + +Downstream jobs (e.g. awips, gempak, etc.) are not included in the diagram. Those jobs are not normally run in developmental tests. + +============================= +Jobs in the GFS Configuration +============================= ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| Job Name | Purpose | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| anal | Runs the analysis. 1) Runs the atmospheric analysis (global_gsi) to produce analysis increments; 2) Update surface | +| | guess file via global_cycle to create surface analysis on tiles. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| analcalc | Adds the analysis increments to previous cycle’s forecasts to produce atmospheric analysis files. Produces surface | +| | analysis file on Gaussian grid. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| analdiag | Creates netCDF diagnostic files containing observation values, innovation (O-F), error, quality control, as well as | +| | other analysis-related quantities (cnvstat, radstat, ozstat files). | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| arch | Archives select files from the deterministic model and cleans up older data. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| earcN/eamn | Archival script for EnKF: 1) Write select EnKF output to HPSS; 2) Copy select files to online archive; 3) Clean up | +| | EnKF temporary run directories; 4) Remove "old" EnKF files from rotating directory. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| ecenN/ecmn | Recenter ensemble members around hi-res deterministic analysis. Note - GFS v16 recenters ensemble member analysis | +| | increments. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| echgres | Runs chgres on full-resolution forecast for EnKF recentering (ecen). | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| ediag | Same as analdiag but for ensemble members. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| efcsN/efmn | Run 9 hour forecast for each ensemble member. There are 80 ensemble members. Each efcs job sequentially processes 8 | +| | ensemble members, so there are 10 efcs jobs in total. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| eobs | Data selection for EnKF update (eupd). | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| eposN/epmn | Generate ensemble mean atmospheric and surface forecast files. The ensemble spread is also computed for atmospheric | +| | forecast files. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| esfc | Generate ensemble surface analyses on tiles. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| eupd | Perform EnKF update (i.e., generate ensemble member analyses). | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| fcst | Runs the forecast (with or without one-way waves). | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| gldas | Runs the Global Land Data Assimilation System (GLDAS). | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| metpN | Runs MET/METplus verification via EMC_verif-global. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| prep | Runs the data preprocessing prior to the analysis (storm relocation if needed and generation of prepbufr file). | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| postN | Runs the post processor. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| vrfy | Runs the verification tasks. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| waveinit | Runs wave initialization step. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| waveprep | Runs wave prep step. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| wavepostsbs | Runs wave post-processing side-by-side. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| wavepostbndpnt | Runs wave post-processing for boundary points. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| wavepostbndpntbll | Runs wave post-processing for boundary points bulletins. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ +| wavepostpnt | Runs wave post-processing for points. | ++-------------------+-----------------------------------------------------------------------------------------------------------------------+ diff --git a/docs/source/monitor_rocoto.rst b/docs/source/monitor_rocoto.rst new file mode 100644 index 0000000000..d7126790b3 --- /dev/null +++ b/docs/source/monitor_rocoto.rst @@ -0,0 +1,105 @@ +================== +Monitor ROCOTO Run +================== + +Click `here `__ to view full rocoto documentation on GitHub + + +^^^^^^^^^^^^^^^^^^ +Using command line +^^^^^^^^^^^^^^^^^^ + +You can use Rocoto commands with arguments to check the status of your experiment. + +Start or continue a run:: + + rocotorun -d /path/to/workflow/database/file -w /path/to/workflow/xml/file + +Check the status of the workflow:: + + rocotostat -d /path/to/workflow/database/file -w /path/to/workflow/xml/file [-c YYYYMMDDCCmm,[YYYYMMDDCCmm,...]] [-t taskname,[taskname,...]] [-s] [-T] + +Note: YYYYMMDDCCmm = YearMonthDayCycleMinute ...where mm/Minute is ’00’ for all cycles currently. + +Check the status of a job:: + + rocotocheck -d /path/to/workflow/database/file -w /path/to/workflow/xml/file -c YYYYMMDDCCmm -t taskname + +Force a task to run (ignores dependencies - USE CAREFULLY!):: + + rocotoboot -d /path/to/workflow/database/file -w /path/to/workflow/xml/file -c YYYYMMDDCCmm -t taskname + +Rerun task(s):: + + rocotorewind -d /path/to/workflow/database/file -w /path/to/workflow/xml/file -c YYYYMMDDCCmm -t taskname + +Set a task to complete (overwrites current state):: + + rocotocomplete -d /path/to/workflow/database/file -w /path/to/workflow/xml/file -c YYYYMMDDCCmm -t taskname + +Several dates and task names may be specified in the same command by adding more -c and -t options. However, lists are not allowed. + +^^^^^^^^^^^^^^^^^ +Use ROCOTO viewer +^^^^^^^^^^^^^^^^^ + +An alternative approach is to use A GUI that was designed to assist with monitoring global workflow experiments that use ROCOTO. It can be found under the ush/rocoto folder in global-workflow. + +***** +Usage +***** + +:: + + ./rocoto_viewer.py -d /path/to/workflow/database/file -w /path/to/workflow/xml/file + +Note 1: Terminal/window must be wide enough to display all experiment information columns, viewer will complain if not. + +Note 2: The viewer requires the full path to the database and xml files if you are not in your EXPDIR when you invoke it. + +********************* +What the viewer shows +********************* + + .. figure:: _static/fv3_rocoto_view.png + + Sample output from Rocoto viewer + +The figure above shows a sample output from a Rocoto viewer for a running experiment. Where: + + * First column: cycle (YYYYMMDDCCmm, YYYY=year, MM=month, DD=day, CC=cycle hour, mm=minute) + * Second column: task name (a "<" symbol indicates a group/meta-task, click "x" when meta-task is selected to expand/collapse) + * Third column: job ID from scheduler + * Fourth column: job state (QUEUED, RUNNING, SUCCEEDED, FAILED, or DEAD) + * Fifth column: exit code (0 if all ended well) + * Sixth column: number of tries/attempts to run job (0 when not yet run or just rewound, 1 when run once successfully, 2+ for multiple tries up to max try value where job is considered DEAD) + * Seventh column: job duration in seconds + +************************** +How to navigate the viewer +************************** + +The rocoto viewer accepts both mouse and keyboard inputs. Click “h” for help menu and more options. + +Available viewer commands:: + + c = get information on selected job + r = rewind (rerun) selected job, group, or cycle + R = run rocotorun + b = boot (forcibly run) selected job or group + -> = right arrow key, advance viewer forward to next cycle + <- = left arrow key, advance viewer backward to previous cycle + Q = quit/exit viewer + +Advanced features: + + * Select multiple tasks at once + + - Click “Enter” on a task to select it, click on other tasks or use the up/down arrows to move to other tasks and click “Enter” to select them as well. + - When you next choose “r” for rewinding the pop-up window will now ask if you are sure you want to rewind all those selected tasks. + + * Rewind entire group or cycle + + - Group - While group/metatask is collapsed (<) click “r” to rewind whole group/metatask. + - Cycle - Use up arrow to move selector up past the first task until the entire left column is highlighted. Click “r” and the entire cycle will be rewound. + diff --git a/docs/source/output.rst b/docs/source/output.rst new file mode 100644 index 0000000000..5ccbbb0fc1 --- /dev/null +++ b/docs/source/output.rst @@ -0,0 +1,20 @@ +############### +Plotting Output +############### + +=============== +Analysis output +=============== + +The `GSI Monitor `_ repository contains a monitoring package called **RadMon**. This package reads the information on the radiances contained in the radstat files, such as quality control flags and departure statistics, and produces a webpage with many plots such as time series of data counts for a particular instrument. You can also directly compare two different experiments with this tool. If there are quantities that you are interested in but the RadMon package is not plotting them for you, you can use the existing RadMon code as a guide for how to read them and plot them yourself. The radstat files contain a wealth of information. + +The RadMon package can be found under the ``src/Radiance_Monitor`` folder within the `GSI Monitor`_. If checked out under global-workflow you will find it under ``gsi_monitor.fd/src/Radiance_Monitor``. + +If you have questions or issues getting the package to work for you please contact the developer of RadMon: Ed Safford (edward.safford@noaa.gov). + +=============== +Forecast output +=============== + +This section will be updated when we have some basic plotting utilities using EMCPY + diff --git a/docs/source/run.rst b/docs/source/run.rst new file mode 100644 index 0000000000..9c5100dcaf --- /dev/null +++ b/docs/source/run.rst @@ -0,0 +1,17 @@ +################### +Run Global Workflow +################### + +Here we will show how you can run an experiment using the Global Workflow. The Global workflow is regularly evolving and the underlying UFS-weather-model that it drives can run many different configurations. So this part of the document will be regularly updated. The workflow as it is configured today can be run as forecast only or cycled (forecast+Data Assimilation). Since cycled mode requires a number of Data Assimilation supporting repositories to be checked out, the instructions for the two modes from initial checkout stage will be slightly different. Apart from this there is a third mode that is rarely used in development mode and is primarily for operational use. This mode switches on specialized post processing needed by the avaiation industry. Since the files associated with this mode are restricted, only few users will have need and/or ability to run in this mode. + +.. toctree:: + :hidden: + + clone.rst + init.rst + setup.rst + configure.rst + start.rst + monitor_rocoto.rst + view.rst + errors_faq.rst diff --git a/docs/source/setup.rst b/docs/source/setup.rst new file mode 100644 index 0000000000..28ed697b0d --- /dev/null +++ b/docs/source/setup.rst @@ -0,0 +1,264 @@ +================ +Experiment Setup +================ + + Global workflow uses a set of scripts to help configure and set up the drivers (also referred to as Workflow Manager) that run the end-to-end system. While currently we use a `ROCOTO `__ based system and that is documented here, an `ecFlow `__ based systm is also under development and will be introduced to the Global Workflow when it is mature. To run the setup scripts, you need to make sure to have a copy of ``python3`` with ``numpy`` available. The easiest way to guarantee this is to load python from the `official hpc-stack installation `_ for the machine you are on: + +Hera:: + + module use -a /contrib/anaconda/modulefiles + module load anaconda/anaconda3-5.3.1 + +Orion:: + + module load python/3.7.5 + +WCOSS2:: + + module load python/3.8.6 + +S4:: + + module load miniconda/3.8-s4 + +If running with Rocoto make sure to have a Rocoto module loaded before running setup scripts: + +Hera:: + + module load rocoto/1.3.3 + +Orion:: + + module load contrib + module load rocoto/1.3.3 + +WCOSS2:: + + module use /apps/ops/test/nco/modulefiles/ + module load core/rocoto/1.3.5 + +S4:: + + module load rocoto/1.3.4 + +^^^^^^^^^^^^^^^^^^^^^^^^ +Free-forecast experiment +^^^^^^^^^^^^^^^^^^^^^^^^ + +Scripts that will be used: + + * workflow/setup_expt.py + * workflow/setup_xml.py + +*************************************** +Step 1: Run experiment generator script +*************************************** + +The following command examples include variables for reference but users should not use environmental variables but explicit values to submit the commands. Exporting variables like EXPDIR to your environment causes an error when the python scripts run. Please explicitly include the argument inputs when running both setup scripts:: + + cd workflow + ./setup_expt.py forecast-only --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC] [--resdet $RESDET] + [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comrot $COMROT] [--expdir $EXPDIR] [--icsdir $ICSDIR] + +where: + + * ``forecast-only`` is the first positional argument that instructs the setup script to produce an experiment directory for forecast only experiments. + * $APP is the target application, one of: + + - ATM: atmosphere-only [default] + - ATMW: atm-wave + - ATMA: atm-aerosols + - S2S: atm-ocean-ice + - S2SW: atm-ocean-ice-wave + - S2SWA: atm-ocean-ice-wave-aerosols + + * $START is the start type (warm or cold [default]) + * $IDATE is the initial start date of your run (first cycle CDATE, YYYYMMDDCC) + * $EDATE is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete + * $PSLOT is the name of your experiment [default: test] + * $CONFIGDIR is the path to the /config folder under the copy of the system you're using [default: $TOP_OF_CLONE/parm/config/] + * $RESDET is the FV3 resolution (i.e. 768 for C768) [default: 384] + * $GFS_CYC is the forecast frequency (0 = none, 1 = 00z only [default], 2 = 00z & 12z, 4 = all cycles) + * $COMROT is the path to your experiment output directory. DO NOT include PSLOT folder at end of path, it’ll be built for you. [default: $HOME] + * $EXPDIR is the path to your experiment directory where your configs will be placed and where you will find your workflow monitoring files (i.e. rocoto database and xml file). DO NOT include PSLOT folder at end of path, it will be built for you. [default: $HOME] + * $ICSDIR is the path to the initial conditions. This is handled differently depending on whether $APP is S2S or not. + + - If $APP is ATM or ATMW, this setting is currently ignored + - If $APP is S2S or S2SW, ICs are copied from the central location to this location and the argument is required + +Examples: + +Atm-only:: + + cd workflow + ./setup_expt.py forecast-only --pslot test --idate 2020010100 --edate 2020010118 --resdet 384 --gfs_cyc 4 --comrot /some_large_disk_area/Joe.Schmo/comrot --expdir /some_safe_disk_area/Joe.Schmo/expdir + +Coupled:: + + cd workflow + ./setup_expt.py forecast-only --app S2SW --pslot coupled_test --idate 2013040100 --edate 2013040100 --resdet 384 --comrot /some_large_disk_area/Joe.Schmo/comrot --expdir /some_safe_disk_area/Joe.Schmo/expdir --icsdir /some_large_disk_area/Joe.Schmo/icsdir + +Coupled with aerosols:: + + cd workflow + ./setup_expt.py forecast-only --app S2SWA --pslot coupled_test --idate 2013040100 --edate 2013040100 --resdet 384 --comrot /some_large_disk_area/Joe.Schmo/comrot --expdir /some_safe_disk_area/Joe.Schmo/expdir --icsdir /some_large_disk_area/Joe.Schmo/icsdir + +**************************************** +Step 2: Set user and experiment settings +**************************************** + +Go to your EXPDIR and check/change the following variables within your config.base now before running the next script: + + * ACCOUNT + * HOMEDIR + * STMP + * PTMP + * ARCDIR (location on disk for online archive used by verification system) + * HPSSARCH (YES turns on archival) + * HPSS_PROJECT (project on HPSS if archiving) + * ATARDIR (location on HPSS if archiving) + +If you are using cycling, also change these: + + * imp_physics from 8 (Thompson) to 11 (GFDL) + * CCPP_SUITE to FV3_GFS_v16 (or another suite that uses GFDL) [#]_ + +.. [#] This is a temporary measure until cycling mode works with Thompson + +Some of those variables will be found within a machine-specific if-block so make sure to change the correct ones for the machine you'll be running on. + +Now is also the time to change any other variables/settings you wish to change in config.base or other configs. `Do that now.` Once done making changes to the configs in your EXPDIR go back to your clone to run the second setup script. See :doc: configure.rst for more information on configuring your run. + +************************************* +Step 3: Run workflow generator script +************************************* + +This step sets up the files needed by the Workflow Manager/Driver. At this moment only ROCOTO configurations are generated:: + + ./setup_xml.py $EXPDIR/$PSLOT + +Example:: + + ./setup_xml.py /some_safe_disk_area/Joe.Schmo/expdir/test + +**************************************** +Step 4: Confirm files from setup scripts +**************************************** + +You will now have a rocoto xml file in your EXPDIR ($PSLOT.xml) and a crontab file generated for your use. Rocoto uses CRON as the scheduler. If you do not have a crontab file you may not have had the rocoto module loaded. To fix this load a rocoto module and then rerun setup_xml.py script again. Follow directions for setting up the rocoto cron on the platform the experiment is going to run on. + +^^^^^^^^^^^^^^^^^ +Cycled experiment +^^^^^^^^^^^^^^^^^ + +Scripts that will be used: + + * workflow/setup_expt.py + * workflow/setup_xml.py + +*************************************** +Step 1) Run experiment generator script +*************************************** + +The following command examples include variables for reference but users should not use environmental variables but explicit values to submit the commands. Exporting variables like EXPDIR to your environment causes an error when the python scripts run. Please explicitly include the argument inputs when running both setup scripts:: + + cd workflow + ./setup_expt.py cycled --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC] + [--resdet $RESDET] [--resens $RESENS] [--nens $NENS] [--cdump $CDUMP] + [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comrot $COMROT] [--expdir $EXPDIR] [--icsdir $ICSDIR] + +where: + + * ``cycled`` is the first positional argument that instructs the setup script to produce an experiment directory for cycled experiments. + * $APP is the target application, one of[#]_: + + - ATM: atmosphere-only [default] + - ATMW: atm-wave + + * $IDATE is the initial start date of your run (first cycle CDATE, YYYYMMDDCC) + * $EDATE is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete + * $START is the start type (warm or cold [default]) + * $GFS_CYC is the forecast frequency (0 = none, 1 = 00z only [default], 2 = 00z & 12z, 4 = all cycles) + * $RESDET is the FV3 resolution of the deterministic forecast [default: 384] + * $RESENS is the FV3 resolution of the ensemble (EnKF) forecast [default: 192] + * $NENS is the number of ensemble members [default: 20] + * $CDUMP is the starting phase [default: gdas] + * $PSLOT is the name of your experiment [default: test] + * $CONFIGDIR is the path to the /config folder under the copy of the system you're using [default: $TOP_OF_CLONE/parm/config/] + * $COMROT is the path to your experiment output directory. DO NOT include PSLOT folder at end of path, it’ll be built for you. [default: $HOME] + * $EXPDIR is the path to your experiment directory where your configs will be placed and where you will find your workflow monitoring files (i.e. rocoto database and xml file). DO NOT include PSLOT folder at end of path, it will be built for you. [default: $HOME] + * $ICSDIR is the path to the ICs for your run if generated separately. [default: None] + +.. [#] More Coupled configurations in cycled mode are currently under development and not yet available + +Example:: + + cd workflow + ./setup_expt.py cycled --pslot test --configdir /home/Joe.Schmo/git/global-workflow/parm/config --idate 2020010100 --edate 2020010118 --comrot /some_large_disk_area/Joe.Schmo/comrot --expdir /some_safe_disk_area/Joe.Schmo/expdir --resdet 384 --resens 192 --nens 80 --gfs_cyc 4 + +Example setup_expt.py on WCOSS_C:: + + SURGE-slogin1 > ./setup_expt.py cycled --pslot fv3demo --idate 2017073118 --edate 2017080106 --comrot /gpfs/hps2/ptmp/Joe.Schmo --expdir /gpfs/hps3/emc/global/noscrub/Joe.Schmo/para_gfs + SDATE = 2017-07-31 18:00:00 + EDATE = 2017-08-01 06:00:00 + EDITED: /gpfs/hps3/emc/global/noscrub/Joe.Schmo/para_gfs/fv3demo/config.base as per user input. + DEFAULT: /gpfs/hps3/emc/global/noscrub/Joe.Schmo/para_gfs/fv3demo/config.base.default is for reference only. + Please verify and delete the default file before proceeding. + SURGE-slogin1 > + +The message about the config.base.default is telling you that you are free to delete it if you wish but it’s not necessary to remove. Your resulting config.base was generated from config.base.default and the default one is there for your information. + +What happens if I run setup_expt.py again for an experiment that already exists?:: + + SURGE-slogin1 > ./setup_expt.py forecast-only --pslot fv3demo --idate 2017073118 + --edate 2017080106 --comrot /gpfs/hps2/ptmp/Joe.Schmo --expdir /gpfs/hps3/emc/global/noscrub/Joe.Schmo/para_gfs + COMROT already exists in /gpfs/hps2/ptmp/Joe.Schmo/fv3demo + Do you wish to over-write COMROT [y/N]: y + EXPDIR already exists in /gpfs/hps3/emc/global/noscrub/Joe.Schmo/para_gfs/fv3demo + Do you wish to over-write EXPDIR [y/N]: y + SDATE = 2017-07-31 18:00:00 + EDATE = 2017-08-01 06:00:00 + EDITED: /gpfs/hps3/emc/global/noscrub/Joe.Schmo/para_gfs/fv3demo/config.base as per user input. + DEFAULT: /gpfs/hps3/emc/global/noscrub/Joe.Schmo/para_gfs/fv3demo/config.base.default is for reference only. + Please verify and delete the default file before proceeding. + +Your COMROT and EXPDIR will be deleted and remade. Be careful with this! + +**************************************** +Step 2: Set user and experiment settings +**************************************** + +Go to your EXPDIR and check/change the following variables within your config.base now before running the next script: + + * ACCOUNT + * HOMEDIR + * STMP + * PTMP + * ARCDIR (location on disk for online archive used by verification system) + * HPSSARCH (YES turns on archival) + * HPSS_PROJECT (project on HPSS if archiving) + * ATARDIR (location on HPSS if archiving) + +Some of those variables will be found within a machine-specific if-block so make sure to change the correct ones for the machine you'll be running on. + +Now is also the time to change any other variables/settings you wish to change in config.base or other configs. `Do that now.` Once done making changes to the configs in your EXPDIR go back to your clone to run the second setup script. See :doc: configure.rst for more information on configuring your run. + + +************************************* +Step 3: Run workflow generator script +************************************* + +This step sets up the files needed by the Workflow Manager/Driver. At this moment only ROCOTO configurations are generated:: + + ./setup_xml.py $EXPDIR/$PSLOT + +Example:: + + ./setup_xml.py /some_safe_disk_area/Joe.Schmo/expdir/test + +**************************************** +Step 4: Confirm files from setup scripts +**************************************** + +You will now have a rocoto xml file in your EXPDIR ($PSLOT.xml) and a crontab file generated for your use. Rocoto uses CRON as the scheduler. If you do not have a crontab file you may not have had the rocoto module loaded. To fix this load a rocoto module and then rerun setup_xml.py script again. Follow directions for setting up the rocoto cron on the platform the experiment is going to run on. + diff --git a/docs/source/start.rst b/docs/source/start.rst new file mode 100644 index 0000000000..7775d2db0c --- /dev/null +++ b/docs/source/start.rst @@ -0,0 +1,45 @@ +============== +Start your run +============== + +Make sure a rocoto module is loaded: ``module load rocoto`` + +If needed check for available rocoto modules on machine: ``module avail rocoto`` or ``module spider rocoto`` + +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Start your run from within your EXPDIR +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:: + + rocotorun -d $PSLOT.db -w $PSLOT.xml + +The first jobs of your run should now be queued or already running (depending on machine traffic). How exciting! + +You'll now have a "logs" folder in both your COMROT and EXPDIR. The EXPDIR log folder contains workflow log files (e.g. rocoto command results) and the COMROT log folder will contain logs for each job (previously known as dayfiles). + +^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Set up your experiment cron +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Note: Orion currently only supports cron on Orion-login-1. Cron support for other login nodes is coming in the future. + +:: + + crontab -e + +or + +:: + + crontab $PSLOT.crontab + +**WARNING: ``crontab $PSLOT.crontab`` command will overwrite existing crontab file on your login node. If running multiple crons recommend editing crontab file with ``crontab -e`` command.** + +Check your crontab settings:: + + crontab -l + +Crontab uses following format:: + + */5 * * * * /path/to/rocotorun -w /path/to/workflow/definition/file -d /path/to/workflow/database/file diff --git a/docs/source/view.rst b/docs/source/view.rst new file mode 100644 index 0000000000..744b4525b1 --- /dev/null +++ b/docs/source/view.rst @@ -0,0 +1,44 @@ +====================== +View Experiment output +====================== + +The output from your run will be found in the ``COMROT/ROTDIR`` you established. This is also where you placed your initial conditions. Within your ``COMROT`` you will have the following directory structure (based on the type of experiment you run): + +^^^^^^^^^^^^^ +Free Forecast +^^^^^^^^^^^^^ + +:: + + gfs.YYYYMMDD/CC/atmos <- contains deterministic long forecast gfs inputs/outputs (atmosphere) + gfs.YYYYMMDD/CC/wave <- contains deterministic long forecast gfs inputs/outputs (wave) + logs/ <- logs for each cycle in the run + vrfyarch/ <- contains files related to verification and archival + +^^^^^^ +Cycled +^^^^^^ + +:: + + enkfgdas.YYYYMMDD/CC/atmos/mem###/ <- contains EnKF inputs/outputs for each cycle and each member + gdas.YYYYMMDD/CC/atmos <- contains deterministic gdas inputs/outputs (atmosphere) + gdas.YYYYMMDD/CC/wave <- contains deterministic gdas inputs/outputs (wave) + gfs.YYYYMMDD/CC/atmos <- contains deterministic long forecast gfs inputs/outputs (atmosphere) + gfs.YYYYMMDD/CC/wave <- contains deterministic long forecast gfs inputs/outputs (wave) + logs/ <- logs for each cycle in the run + vrfyarch/ <- contains files related to verification and archival + +Here is an example ``COMROT`` for a cycled run as it may look several cycles in (note the archival steps remove older cycle folders as the run progresses):: + + -bash-4.2$ ll /scratch1/NCEPDEV/stmp4/Joe.Schmo/comrot/testcyc192 + total 88 + drwxr-sr-x 4 Joe.Schmo stmp 4096 Oct 22 04:50 enkfgdas.20190529 + drwxr-sr-x 4 Joe.Schmo stmp 4096 Oct 22 07:20 enkfgdas.20190530 + drwxr-sr-x 6 Joe.Schmo stmp 4096 Oct 22 03:15 gdas.20190529 + drwxr-sr-x 4 Joe.Schmo stmp 4096 Oct 22 07:15 gdas.20190530 + drwxr-sr-x 6 Joe.Schmo stmp 4096 Oct 22 03:15 gfs.20190529 + drwxr-sr-x 4 Joe.Schmo stmp 4096 Oct 22 07:15 gfs.20190530 + drwxr-sr-x 120 Joe.Schmo stmp 12288 Oct 22 07:15 logs + drwxr-sr-x 13 Joe.Schmo stmp 4096 Oct 22 07:07 vrfyarch +