Update develop-ref after #2117 (#2122)

Co-authored-by: Julie Prestopnik <jpresto@seneca.rap.ucar.edu>
Co-authored-by: johnhg <johnhg@ucar.edu>
Co-authored-by: Seth Linden <linden@kiowa.rap.ucar.edu>
Co-authored-by: John Halley Gotway <johnhg@ucar.edu>
Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com>
Co-authored-by: Howard Soh <hsoh@kiowa.rap.ucar.edu>
Co-authored-by: John Halley Gotway <johnhg@kiowa.rap.ucar.edu>
Co-authored-by: jprestop <jpresto@ucar.edu>
Co-authored-by: Howard Soh <hsoh@seneca.rap.ucar.edu>
Co-authored-by: Randy Bullock <bullock@seneca.rap.ucar.edu>
Co-authored-by: davidfillmore <fillmore.winslow.david@gmail.com>
Co-authored-by: rgbullock <bullock@ucar.edu>
Co-authored-by: Seth Linden <linden@ucar.edu>
Co-authored-by: hsoh-u <hsoh@ucar.edu>
Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Co-authored-by: John Halley Gotway <johnhg@seneca.rap.ucar.edu>
Co-authored-by: MET Tools Test Account <met_test@seneca.rap.ucar.edu>
Co-authored-by: mo-mglover <78152252+mo-mglover@users.noreply.github.com>
Co-authored-by: davidalbo <dave@ucar.edu>
Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com>
Co-authored-by: Dan Adriaansen <dadriaan@ucar.edu>

.github/jobs/set_job_controls.sh

@@ -17,6 +17,11 @@ if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
 
     run_diff=true
 
+    # if base branch starts with main_v, use that input data
+    if [ "${GITHUB_BASE_REF:0:6}" == "main_v" ]; then
+	input_data_version=${GITHUB_BASE_REF:6}
+    fi
+
   fi
 
 elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then
@@ -30,6 +35,11 @@ elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then
     run_diff=true
     truth_data_version=${branch_name:0: -4}
 
+    # if branch starts with main_v, use that input data
+    if [ "${branch_name:0:6}" == "main_v" ]; then
+      input_data_version=${branch_name:6:-4}
+    fi
+
   else
 
     # if develop or main_vX.Y branch, run diff tests using branch's truth data
@@ -39,6 +49,11 @@ elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then
        run_diff=true
        truth_data_version=${branch_name}
 
+      # if branch starts with main_v, use that input data
+      if [ "${branch_name:0:6}" == "main_v" ]; then
+        input_data_version=${branch_name:6}
+      fi
+
     fi
 
     # check commit messages for skip or force keywords                                                                         
@@ -66,7 +81,13 @@ elif [ "${GITHUB_EVENT_NAME}" == "push" ]; then
     fi
 
   fi
-
+
+elif [ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]; then
+
+    if [ "${force_tests}" == "true" ]; then
+        run_diff=true
+    fi
+
 fi
 
 # if updating truth or running diff, run unit tests

.github/workflows/build_docker_and_trigger_metplus.yml

@@ -25,13 +25,13 @@ jobs:
       - name: Build Docker Image
         run: .github/jobs/build_docker_image.sh
         env:
-          SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}
+          SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}-lite
           MET_BASE_IMAGE: minimum
 
       - name: Push Docker Image
         run: .github/jobs/push_docker_image.sh
         env:
-          SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}
+          SOURCE_BRANCH: ${{ steps.get_branch_name.outputs.branch_name }}-lite
           DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
 

.github/workflows/testing.yml

@@ -23,6 +23,13 @@ on:
     paths-ignore:
       - 'met/docs/**'
 
+  workflow_dispatch:
+    inputs:
+      force_tests:
+        description: 'Run the unit tests'
+        default: true
+        type: boolean
+
 env:
   DOCKERHUB_REPO: dtcenter/met-dev
 
@@ -39,6 +46,7 @@ jobs:
         run: .github/jobs/set_job_controls.sh
         env:
           commit_msg: ${{ github.event.head_commit.message }}
+          force_tests: ${{ github.event.inputs.force_tests }}
 
     outputs:
       run_compile: ${{ steps.job_status.outputs.run_compile }}

met/README

@@ -188,7 +188,7 @@ sub-directory, or the MET Online Tutorial:
 
 - If all tools are enabled and the build is successful, the "<prefix>/bin"
   directory (where <prefix> is the prefix you specified on your configure
-  command line) will contain 36 executables:
+  command line) will contain the following executables:
    - ascii2nc
    - ensemble_stat
    - gen_ens_prod
@@ -200,6 +200,7 @@ sub-directory, or the MET Online Tutorial:
    - grid_diag
    - gsid2mpr
    - gsidens2orank
+   - ioda2nc
    - lidar2nc
    - madis2nc
    - mode

met/data/config/IODA2NCConfig_default

@@ -125,6 +125,6 @@ time_summary = {
 ////////////////////////////////////////////////////////////////////////////////
 
 tmp_dir = "/tmp";
-version = "V10.0";
+version = "V10.1.0";
 
 ////////////////////////////////////////////////////////////////////////////////

met/docs/Users_Guide/appendixF.rst

@@ -236,10 +236,11 @@ The Ensemble-Stat, Series-Analysis, and MTD tools support the use of file lists
      file_type=PYTHON_NUMPY;' \
     -title "Python enabled plot_data_plane"
 
+.. _pyembed-point-obs-data:
+
 Python Embedding for Point Observations
 =======================================
 
-
 The ASCII2NC tool supports the "-format python" option. With this option, point observations may be passed as input. An example of this is provided in :numref:`ascii2nc-pyembed`. That example uses the **read_ascii_point.py** sample script which is included with the MET code. It reads ASCII data in MET's 11-column point observation format and stores it in a Pandas dataframe to be read by the ASCII2NC tool with Python.
 
 The **read_ascii_point.py** sample script can be found in:
@@ -248,33 +249,13 @@ The **read_ascii_point.py** sample script can be found in:
 
 • `MET GitHub repository <https://github.com/dtcenter/MET>`_ in *met/scripts/python*.
 
-Python Embedding for MPR data
-=============================
-
-The Stat-Analysis tool supports the "-lookin python" option. With this option, matched pair (MPR) data may be passed as input. An example of this is provided in :numref:`StA-pyembed`. That example uses the **read_ascii_mpr.py** sample script which is included with the MET code. It reads MPR data and stores it in a Pandas dataframe to be read by the Stat-Analysis tool with Python.
-
-The **read_ascii_mpr.py** sample script can be found in:
-
-• MET installation directory in *MET_BASE/python*.
-
-• `MET GitHub repository <https://github.com/dtcenter/MET>`_ in *met/scripts/python*.
-
-
-Python Embedding for Point Observations as input
-================================================
-
-
-The point2grid, plot_point_obs, ensemble_stat, and point_stat tools use MET point observation NetCDF. They support the python embedding by the prefix 'PYTHON_NUMPY=" and followed by a python script name instead of the MET point observastion NetCDF filename. The customized python script is expected to extend MET_BASE/python/met_point_obs.py and to produce the python variable, **met_point_data**,  which is the dictionary of the MET point observation data. They are defined at MET_BASE/python/met_point_obs.py.
-
-
-.. _pyembed-point-obs-data:
-
+The Point2Grid, Plot-Point-Obs, Ensemble-Stat, and Point-Stat tools also process point observations. They support python embedding of point observations directly on the command line by replacing the input MET NetCDF point observation file name with the python command to be run. The command must begin with the prefix 'PYTHON_NUMPY=' and be followed by the path to python script and any arguments. The full command should be enclosed in single quotes to prevent embedded whitespace from causing parsing errors. The customized python script is expected to extend MET_BASE/python/met_point_obs.py. That script creates a python variable named **met_point_data** which is a dictionary containing formatted point observation data.
 
 .. code-block:: none
 
   met_point_data = {
 
-     'use_var_id':  Trur/False,     # obs_vid are variable index if True, otherwise GRIB codes
+     'use_var_id':  True/False,     # obs_vid are variable index if True, otherwise GRIB codes
 
      # Header data
      'nhdr':        integer_value,  # number of headers
@@ -308,3 +289,14 @@ The point2grid, plot_point_obs, ensemble_stat, and point_stat tools use MET poin
      'obs_qty_table':  string_array,    # quality marks
      'obs_var_table':  string_array,    # variable names
   }
+
+Python Embedding for MPR data
+=============================
+
+The Stat-Analysis tool supports the "-lookin python" option. With this option, matched pair (MPR) data may be passed as input. An example of this is provided in :numref:`StA-pyembed`. That example uses the **read_ascii_mpr.py** sample script which is included with the MET code. It reads MPR data and stores it in a Pandas dataframe to be read by the Stat-Analysis tool with Python.
+
+The **read_ascii_mpr.py** sample script can be found in:
+
+• MET installation directory in *MET_BASE/python*.
+
+• `MET GitHub repository <https://github.com/dtcenter/MET>`_ in *met/scripts/python*.

met/docs/Users_Guide/ensemble-stat.rst

@@ -9,6 +9,8 @@ Introduction
 
 The Ensemble-Stat tool may be run to create simple ensemble forecasts (mean, probability, spread, etc) from a set of several forecast model files to be used by the MET statistics tools. If observations are also included, ensemble statistics such as rank histograms, probability integral transform histograms, spread/skill variance, relative position and continuous ranked probability score are produced. Climatological mean and standard deviation data may also be provided, and will be used as a reference forecast in several of the output statistics. Finally, observation error perturbations can be included prior to calculation of statistics. Details about and equations for the statistics produced for ensembles are given in :numref:`Appendix C, Section %s <App_C-ensemble>`.
 
+.. note:: This tool will be changing! The ensemble product generation step provided by Ensemble-Stat is now found within the :ref:`Gen-Ens-Prod Tool<gen-ens-prod>`. The Gen-Ens-Prod tool replaces and extends that functionality. Users are strongly encouraged to migrate ensemble product generation from Ensemble-Stat to Gen-Ens-Prod, as new features will only be added to Gen-Ens-Prod and the existing Ensemble-Stat functionality will be deprecated in a future version.
+
 Scientific and statistical aspects
 ==================================
 
@@ -100,7 +102,7 @@ Optional arguments for ensemble_stat
 
 4. To produce ensemble statistics using gridded observations, use the **-grid_obs file** option to specify a gridded observation file. This option may be used multiple times if your observations are in several files.
 
-5. To produce ensemble statistics using point observations, use the **-point_obs file** option to specify a NetCDF point observation file. This option may be used multiple times if your observations are in several files. The python embedding will be activated if the **file** begines with 'PYTHON_NUMPY=" and followed by a python script name.
+5. To produce ensemble statistics using point observations, use the **-point_obs file** option to specify a NetCDF point observation file. This option may be used multiple times if your observations are in several files. Python embedding for point observations is also supported, as described in :numref:`pyembed-point-obs-data`.
 
 6. To override the simple ensemble mean value of the input ensemble members for the ECNT, SSVAR, and ORANK line types, the **-ens_mean file** option specifies an ensemble mean model data file. This option replaces the **-ssvar_mean file** option from earlier versions of MET.
 

met/docs/Users_Guide/gen-ens-prod.rst

@@ -7,9 +7,9 @@ Gen-Ens-Prod Tool
 Introduction
 ============
 
-The Gen-Ens-Prod tool generates simple ensemble products (mean, spread, probability, etc) from gridded ensemble member input files. While it processes model inputs, but it does not compare them to observations or compute statistics. However, the output products can be passed as input to the MET statistics tools for comparison against observations. Climatological mean and standard deviation data may also be provided to define thresholds based on the climatological distribution at each grid point.
+The Gen-Ens-Prod tool generates simple ensemble products (mean, spread, probability, etc) from gridded ensemble member input files. While it processes model inputs, it does not compare them to observations or compute statistics. However, the output products can be passed as input to the MET statistics tools for comparison against observations. Climatological mean and standard deviation data may also be provided to define thresholds based on the climatological distribution at each grid point.
 
-Note that this ensemble product generation step was provided by the Ensemble-Stat tool in earlier versions of MET. The Gen-Ens-Prod tool replaces and extends that functionality. Users are strongly encouraged to migrate ensemble product generation from Ensemble-Stat to Gen-Ens-Prod, as new features will only be added to Gen-Ens-Prod and the existing Ensemble-Stat functionality will be deprecated in a future version.
+.. note:: This ensemble product generation step was provided by the Ensemble-Stat tool in earlier versions of MET. The Gen-Ens-Prod tool replaces and extends that functionality. Users are strongly encouraged to migrate ensemble product generation from Ensemble-Stat to Gen-Ens-Prod, as new features will only be added to Gen-Ens-Prod and the existing Ensemble-Stat functionality will be deprecated in a future version.
 
 Scientific and statistical aspects
 ==================================