Feature 1561 use case ugrid python embedding (#1927)

Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com>

.github/jobs/docker_setup.sh

@@ -67,7 +67,7 @@ fi
 
 
 # skip docker push if credentials are not set
-if [ -z ${DOCKER_USERNAME+x} ] || [ -z ${DOCKER_PASSWORD+x} ]; then
+if [ -z "${DOCKER_USERNAME}" ] || [ -z "${DOCKER_PASSWORD}" ]; then
     echo "DockerHub credentials not set. Skipping docker push"
     exit 0
 fi

.github/parm/use_case_groups.json

@@ -218,5 +218,10 @@
     "category": "tc_and_extra_tc",
     "index_list": "3-5",
     "run": false
+  },
+  {
+    "category": "unstructured_grids",
+    "index_list": "0",
+    "run": false
   }
 ]

docs/use_cases/model_applications/unstructured_grids/README.rst

@@ -0,0 +1,4 @@
+Unstructured Grids
+------------------
+
+Unstructured grids used by models for numerical weather prediction.

docs/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.py

@@ -0,0 +1,160 @@
+"""
+StatAnalysis: UGRID
+===========================================================================
+
+model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf
+
+"""
+
+###########################################
+# Scientific Objective
+# --------------------
+#
+# This use case demonstrates the use of python embedding to ingest and perform
+# verification on an unstructured grid. This foregoes the need to interpolate
+# to a regular grid as a step in the verification process, thereby avoiding
+# any incurred interpolation error in the process.
+#
+# In particular, this use case ingests a UK MET Office LFRic forecast file in
+# NetCDF format, which resides in the UGRID format of the cubed-sphere. The python
+# library Iris was developed to perform analysis on various UGRID formats, and is
+# employed here to ingest the file as well as perform direct interpolation 
+# from the native forecast grid to observation locations, thereby forming matched
+# pairs to pass to stat_analysis. In order to perform the interpolation using a 
+# nearest-neighbors approach, the geovista python package is also used to form a 
+# KD tree to be used in identifying the interpolation points to be used. This
+# package is located at https://github.com/bjlittle/geovista/ and can be installed
+# from a development version. It is also required to install the pyvista python 
+# package. ASCII files containing observations are also ingested.
+#
+# The python embedding script itself performs the interpolation in time, and
+# for this use case thins the observation data in order to reduce the run time.
+# It is also noted that the observations for this use case were fabricated and
+# correlated observation-forecast pairs are not expected.
+#
+
+##############################################################################
+# Datasets
+# --------
+#
+#
+# | **Data source:** UK MET Office LFRic forecast files in UGRID NetCDF format and observations in ASCII format
+#
+# | **Location:** All of the input data required for this use case can be found in the met_test sample data tarball. Click here to the METplus releases page and download sample data for the appropriate release: https://github.com/dtcenter/METplus/releases
+# | The tarball should be unpacked into the directory that you will set the value of INPUT_BASE. See `Running METplus`_ section for more information.
+# |
+
+##############################################################################
+# METplus Components
+# ------------------
+#
+# This use case utilizes the METplus StatAnalysis wrapper to search for
+# files that are valid for the given case and generate a command to run
+# the MET tool stat_analysis.
+
+##############################################################################
+# METplus Workflow
+# ----------------
+#
+# StatAnalysis is the only tool called in this example. It processes the following
+# run times:
+#
+# | **Valid:** 2021-05-05_00Z  
+# | **Forecast lead:** 12 hour
+# |
+
+##############################################################################
+# METplus Configuration
+# ---------------------
+#
+# METplus first loads all of the configuration files found in parm/metplus_config,
+# then it loads any configuration files passed to METplus via the command line
+# with the -c option, i.e. -c parm/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf
+#
+# .. highlight:: bash
+# .. literalinclude:: ../../../../parm/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf
+
+##############################################################################
+# MET Configuration
+# -----------------
+#
+# METplus sets environment variables based on user settings in the METplus configuration file. 
+# See :ref:`How METplus controls MET config file settings<metplus-control-met>` for more details. 
+#
+# **YOU SHOULD NOT SET ANY OF THESE ENVIRONMENT VARIABLES YOURSELF! THEY WILL BE OVERWRITTEN BY METPLUS WHEN IT CALLS THE MET TOOLS!**
+#
+# If there is a setting in the MET configuration file that is currently not supported by METplus you'd like to control, please refer to:
+# :ref:`Overriding Unsupported MET config file settings<met-config-overrides>`
+#
+# .. note:: See the :ref:`StatAnalysis MET Configuration<stat-analysis-met-conf>` section of the User's Guide for more information on the environment variables used in the file below:
+#
+# .. highlight:: bash
+# .. literalinclude:: ../../../../parm/met_config/STATAnalysisConfig_wrapped
+
+##############################################################################
+# Python Embedding
+# ----------------
+#
+# This use case uses a Python embedding script to read input data
+#
+# parm/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed/ugrid_lfric_mpr.py
+#
+# .. highlight:: python
+# .. literalinclude:: ../../../../parm/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed/ugrid_lfric_mpr.py
+#
+
+##############################################################################
+# Running METplus
+# ---------------
+#
+# It is recommended to run this use case by:
+#
+# Passing in StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf then a user-specific system configuration file::
+#
+#   run_metplus.py -c /path/to/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf -c /path/to/user_system.conf
+#
+# The following METplus configuration variables must be set correctly to run this example.:
+#
+# * **INPUT_BASE** - Path to directory where sample data tarballs are unpacked (See Datasets section to obtain tarballs).
+# * **OUTPUT_BASE** - Path where METplus output will be written. This must be in a location where you have write permissions
+# * **MET_INSTALL_DIR** - Path to location where MET is installed locally
+#
+# Example User Configuration File::
+#
+#   [dir]
+#   INPUT_BASE = /path/to/sample/input/data
+#   OUTPUT_BASE = /path/to/output/dir
+#   MET_INSTALL_DIR = /path/to/met-X.Y 
+#
+# **NOTE:** All of these items must be found under the [dir] section.
+#
+
+
+##############################################################################
+# Expected Output
+# ---------------
+#
+# A successful run will output the following both to the screen and to the logfile::
+#
+#   INFO: METplus has successfully finished running.
+#
+# Refer to the value set for **OUTPUT_BASE** to find where the output data was generated.
+# Output for this use case will be found in StatAnalysis_UGRID  (relative to **OUTPUT_BASE**)
+# and will contain the following file:
+#
+# * dump.out
+
+##############################################################################
+# Keywords
+# --------
+#
+# .. note::
+#
+#   * StatAnalysisToolUseCase 
+#   * PythonEmbeddingFileUseCase 
+#   * UnstructureGridsUseCase
+#
+#   Navigate to the :ref:`quick-search` page to discover other similar use cases.
+#
+#
+# sphinx_gallery_thumbnail_path = '_static/unstructured_grids-StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.png'

internal/tests/use_cases/all_use_cases.txt

@@ -166,3 +166,7 @@ Category: tc_and_extra_tc
 3::GridStat_fcstHAFS_obsTDR_NetCDF:: model_applications/tc_and_extra_tc/GridStat_fcstHAFS_obsTDR_NetCDF.conf:: py_embed
 4::TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample:: model_applications/tc_and_extra_tc/TCPairs_TCStat_fcstADECK_obsBDECK_ATCF_BasicExample.conf
 5::TCGen_fcstGFS_obsBDECK_2021season:: model_applications/tc_and_extra_tc/TCGen_fcstGFS_obsBDECK_2021season.conf
+
+
+Category: unstructured_grids
+0::StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed:: model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf:: geovista_env, py_embed

parm/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.conf

@@ -0,0 +1,85 @@
+[config]
+
+# Documentation for this use case can be found at
+# https://metplus.readthedocs.io/en/latest/generated/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed.html
+
+# For additional information, please see the METplus Users Guide.
+# https://metplus.readthedocs.io/en/latest/Users_Guide
+
+###
+# Processes to run
+# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#process-list
+###
+
+PROCESS_LIST = StatAnalysis
+
+
+###
+# Time Info
+# LOOP_BY options are INIT, VALID, RETRO, and REALTIME
+# If set to INIT or RETRO:
+#   INIT_TIME_FMT, INIT_BEG, INIT_END, and INIT_INCREMENT must also be set
+# If set to VALID or REALTIME:
+#   VALID_TIME_FMT, VALID_BEG, VALID_END, and VALID_INCREMENT must also be set
+# LEAD_SEQ is the list of forecast leads to process
+# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#timing-control
+###
+
+LOOP_BY = VALID
+
+VALID_TIME_FMT = %Y%m%d%H
+VALID_BEG=2021050500
+VALID_END=2021050500
+VALID_INCREMENT = 6H
+
+LEAD_SEQ = 0
+
+
+###
+# File I/O
+# https://metplus.readthedocs.io/en/latest/Users_Guide/systemconfiguration.html#directory-and-filename-template-info
+###
+
+MODEL1_STAT_ANALYSIS_LOOKIN_DIR = python {PARM_BASE}/use_cases/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed/ugrid_lfric_mpr.py {INPUT_BASE}/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed/fcst_data/lfric_ver_20210505_0000.nc {INPUT_BASE}/model_applications/unstructured_grids/StatAnalysis_fcstLFRIC_UGRID_obsASCII_PyEmbed/obs_data
+
+STAT_ANALYSIS_OUTPUT_DIR = {OUTPUT_BASE}/StatAnalysis_UGRID
+STAT_ANALYSIS_OUTPUT_TEMPLATE = job.out
+MODEL1_STAT_ANALYSIS_DUMP_ROW_TEMPLATE = dump.out
+
+
+###
+# StatAnalysis Settings
+# https://metplus.readthedocs.io/en/latest/Users_Guide/wrappers.html#statanalysis
+###
+
+MODEL1 = NA
+MODEL1_OBTYPE = NA
+
+STAT_ANALYSIS_JOB_NAME = aggregate_stat
+STAT_ANALYSIS_JOB_ARGS = -out_line_type CNT -dump_row [dump_row_file] -line_type MPR -by FCST_VAR
+
+MODEL_LIST =
+DESC_LIST =
+FCST_LEAD_LIST =
+OBS_LEAD_LIST =
+FCST_VALID_HOUR_LIST = 
+FCST_INIT_HOUR_LIST =
+OBS_VALID_HOUR_LIST =
+OBS_INIT_HOUR_LIST =
+FCST_VAR_LIST =
+OBS_VAR_LIST =
+FCST_UNITS_LIST =
+OBS_UNITS_LIST =
+FCST_LEVEL_LIST =
+OBS_LEVEL_LIST =
+VX_MASK_LIST =
+INTERP_MTHD_LIST =
+INTERP_PNTS_LIST =
+FCST_THRESH_LIST =
+OBS_THRESH_LIST =
+COV_THRESH_LIST =
+ALPHA_LIST =
+LINE_TYPE_LIST =
+
+GROUP_LIST_ITEMS =
+LOOP_LIST_ITEMS = MODEL_LIST