Update Develop-ref after #1348 (#1355)

Co-authored-by: George McCabe <23407799+georgemccabe@users.noreply.github.com>
Co-authored-by: Julie Prestopnik <jpresto@seneca.rap.ucar.edu>
Co-authored-by: TaraJensen <jensen@ucar.edu>
Co-authored-by: Mrinal Biswas <biswas@ucar.edu>
Co-authored-by: bikegeek <3753118+bikegeek@users.noreply.github.com>
Co-authored-by: bikegeek <minnawin@ucar.edu>
Co-authored-by: johnhg <johnhg@ucar.edu>
Co-authored-by: Christina Kalb <kalb@ucar.edu>
Co-authored-by: lisagoodrich <33230218+lisagoodrich@users.noreply.github.com>
Co-authored-by: jprestop <jpresto@ucar.edu>
Co-authored-by: j-opatz <59586397+j-opatz@users.noreply.github.com>

.github/jobs/get_use_case_commands.py

@@ -129,7 +129,8 @@ def main(categories, subset_list, work_dir=None,
 
             setup_env, py_embed_arg = handle_automation_env(host_name, reqs, work_dir)
 
-            use_case_cmds = []
+            # use status variable to track if any use cases failed
+            use_case_cmds = ['status=0']
             for use_case in use_case_by_requirement.use_cases:
                 # add parm/use_cases path to config args if they are conf files
                 config_args = []
@@ -147,7 +148,12 @@ def main(categories, subset_list, work_dir=None,
                                 f" {py_embed_arg}{test_settings_conf}"
                                 f" config.OUTPUT_BASE={output_base}")
                 use_case_cmds.append(use_case_cmd)
+                # check exit code from use case command and
+                # set status to non-zero value on error
+                use_case_cmds.append("if [ $? != 0 ]; then status=1; fi")
 
+            # if any use cases failed, force non-zero exit code with false
+            use_case_cmds.append("if [ $status != 0 ]; then false; fi")
             # add commands to set up environment before use case commands
             group_commands = f"{setup_env}{';'.join(use_case_cmds)}"
             all_commands.append((group_commands, reqs))

.github/jobs/run_diff_docker.py

@@ -13,19 +13,16 @@
 import shutil
 
 GITHUB_WORKSPACE = os.environ.get('GITHUB_WORKSPACE')
-# add ci/util to sys path to get diff utility
+# add util directory to sys path to get diff utility
 diff_util_dir = os.path.join(GITHUB_WORKSPACE,
-                             'ci',
+                             'metplus',
                              'util')
 sys.path.insert(0, diff_util_dir)
 from diff_util import compare_dir
 
 TRUTH_DIR = '/data/truth'
 OUTPUT_DIR = '/data/output'
 DIFF_DIR = '/data/diff'
-# DIFF_DIR = os.path.join(GITHUB_WORKSPACE,
-#                         'artifact',
-#                         'diff')
 
 def copy_diff_output(diff_files):
     """!  Loop through difference output and copy files

.github/jobs/set_job_controls.sh

@@ -13,6 +13,7 @@ run_use_cases=true
 run_save_truth_data=false
 run_all_use_cases=false
 run_diff=false
+external_trigger=false
 
 # run all use cases and diff logic for pull request
 if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
@@ -25,6 +26,12 @@ if [ "${GITHUB_EVENT_NAME}" == "pull_request" ]; then
     run_all_use_cases=true
     run_diff=true
   fi
+# run all use cases and diff logic for external workflow trigger
+elif [ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]; then
+    run_use_cases=true
+    run_all_use_cases=true
+    run_diff=true
+    external_trigger=true
 # run all use cases and save truth data if -ref branch and not PR
 elif [ "${GITHUB_REF: -4}" == -ref ]; then
   run_use_cases=true
@@ -98,13 +105,15 @@ echo run_use_cases=${run_use_cases} >> job_control_status
 echo run_save_truth_data=${run_save_truth_data} >> job_control_status
 echo run_all_use_cases=${run_all_use_cases} >> job_control_status
 echo run_diff=${run_diff} >> job_control_status
+echo external_trigger=${external_trigger} >> job_control_status
 echo Job Control Settings:
 cat job_control_status
 
 echo ::set-output name=run_get_image::$run_get_image
 echo ::set-output name=run_get_input_data::$run_get_input_data
 echo ::set-output name=run_diff::$run_diff
 echo ::set-output name=run_save_truth_data::$run_save_truth_data
+echo ::set-output name=external_trigger::$external_trigger
 
 # get use cases to run
 .github/jobs/get_use_cases_to_run.sh $run_use_cases $run_all_use_cases $run_unit_tests

.github/parm/use_case_groups.json

@@ -59,6 +59,11 @@
     "index_list": "0-2",
     "run": false
   },
+  {
+    "category": "marine_and_cryosphere",
+    "index_list": "3",
+    "run": false
+  },
   {
     "category": "medium_range",
     "index_list": "0",

.github/workflows/testing.yml

@@ -13,8 +13,31 @@ on:
     types: [opened, reopened, synchronize]
     paths-ignore:
       - docs/** 
+  workflow_dispatch:
+    inputs:
+      repository:
+        description: 'Repository that triggered workflow'
+        required: true
+      sha:
+        description: 'Commit hash that triggered the event'
+        required: true
+      ref:
+        description: 'Branch that triggered event'
+      actor:
+        description: 'User that triggered the event'
+      pusher_email:
+        description: 'Email address of user who triggered push event'
 
 jobs:
+  event_info:
+    name: "Trigger: ${{ github.event_name != 'workflow_dispatch' && github.event_name || github.event.inputs.repository }} ${{ github.event_name != 'workflow_dispatch' && 'local' || github.event.inputs.pusher_email }} ${{ github.event_name != 'workflow_dispatch' && 'event' || github.event.inputs.sha }}"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Print GitHub values for reference
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+
   job_control:
     name: Determine which jobs to run
     runs-on: ubuntu-latest
@@ -25,12 +48,9 @@ jobs:
       run_get_input_data: ${{ steps.job_status.outputs.run_get_input_data }}
       run_diff: ${{ steps.job_status.outputs.run_diff }}
       run_save_truth_data: ${{ steps.job_status.outputs.run_save_truth_data }}
+      external_trigger: ${{ steps.job_status.outputs.external_trigger }}
     steps:
       - uses: actions/checkout@v2
-      - name: Print GitHub values for reference
-        env:
-          GITHUB_CONTEXT: ${{ toJson(github) }}
-        run: echo "$GITHUB_CONTEXT"
       - name: Set job controls
         id: job_status
         run: .github/jobs/set_job_controls.sh

docs/Contributors_Guide/add_use_case.rst

@@ -144,8 +144,20 @@ Use Case Rules
 - The use case should be run by someone other than the author to ensure that it
   runs smoothly outside of the development environment set up by the author.
 
-.. _use_case_documentation:
+.. _memory-intense-use-cases:
+
+Use Cases That Exceed Github Actions Memory Limit
+-------------------------------------------------
+
+Below is a list of use cases in the repository that cannot be run in Github Actions 
+due to their excessive memory usage. They have been tested and cleared by reviewers 
+of any other issues and can be used by METplus users in the same manner as all 
+other use cases.
 
+- model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst
+
+.. _use_case_documentation:
+
 Document New Use Case
 ---------------------
 
@@ -1024,6 +1036,24 @@ with "Use Case Tests." Click on the job and search for the use case config
 filename in the log output by using the search box on the top right of the
 log output.
 
+If the use case fails in GitHub Actions but runs successfully in the user's environment, 
+potential reasons include: 
+
+- Errors providing input data (see :ref:`use_case_input_data`)
+- Using hard-coded paths from the user's machine
+- Referencing variables set in the user's configuration file or local environment
+- Memory usuage of the use case exceeds the available memory in hte Github Actions environment
+
+Github Actions has `limited memory <https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources>`_
+available and will cause the use case to fail when exceeded. A failure caused by exceeding 
+the memory allocation in a Python Embedding script may result in an unclear error message. 
+If you suspect that this is the case, consider utilizing a Python memory profiler to check the
+Python script's memory usage. If your use case exceeds the limit, try to pare 
+down the data held in memory and use less memory intensive Python routines.
+
+If memory mitigation cannot move the use case’s memory usage below the Github Actions limit, 
+see :ref:`exceeded-Github-Actions` for next steps.
+
 Verify that the use case ran in a reasonable amount of time
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -1039,6 +1069,40 @@ run the set of use cases is now above 20 minutes or so, consider creating a
 new job for the new use case. See the :ref:`subset_category` section and the
 multiple medium_range jobs for an example.
 
+
+.. _exceeded-Github-Actions:
+
+Use Cases That Exceed Memory Allocations of Github Actions
+----------------------------------------------------------
+
+If a use case utilizing Python embedding does not run successfully in 
+Github Actions due to exceeding the memory limit and memory mitigation 
+steps were unsuccessful in lowering memory usage, please take the following steps.
+
+- Document the Github Actions failure in the Github use case issue. 
+  Utilize a Python memory profiler to identify as specifically as possible 
+  where the script exceeds the memory limit.
+- Add the use case to the :ref:`memory-intense-use-cases` list.
+- In the internal_tests/use_cases/all_use_cases.txt file, ensure that the 
+  use case is listed as the lowest-listed use case in its respective category. 
+  Change the number in front of the new use case to an 'X', preceeded 
+  by the ‘#’ character::
+
+	#X::GridStat_fcstRTOFS_obsGHRSST_climWOA_sst::model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst.conf, model_applications/marine_and_cryosphere/GridStat_fcstRTOFS_obsGHRSST_climWOA_sst/ci_overrides.conf:: icecover_env, py_embed
+
+- In the **.github/parm/use_case_groups.json** file, remove the entry that 
+  was added during the :ref:`add_new_category_to_test_runs` 
+  for the new use case. This will stop the use case from running on a pull request. 
+- Push these two updated files to your branch in Github and confirm that it 
+  now compiles successfully.
+- During the :ref:`create-a-pull-request` creation, inform the reviewer of 
+  the Github Actions failure. The reviewer should confirm the use case is 
+  successful when run manually, that the memory profiler output confirms that 
+  the Python embedding script exceeds the Github Actions limit, and that 
+  there are no other Github Actions compiling errors.
+
+.. _create-a-pull-request:
+
 Create a Pull Request
 =====================
 

docs/Contributors_Guide/basic_components.rst

@@ -274,7 +274,7 @@ should be set.
 Add Support for MET Dictionary
 ------------------------------
 
-The handle_met_config_dict function can be used to easily set a MET config
+The add_met_config_dict function can be used to easily set a MET config
 dictionary variable. The function takes 2 arguments:
 
 * dict_name: Name of the MET dictionary variable, i.e. distance_map.
@@ -285,7 +285,7 @@ dictionary variable. The function takes 2 arguments:
 
 ::
 
-    self.handle_met_config_dict('fcst_genesis', {
+    self.add_met_config_dict('fcst_genesis', {
         'vmax_thresh': 'thresh',
         'mslp_thresh': 'thresh',
     })
@@ -302,12 +302,12 @@ data type, extra info, children, and nicknames.
 * extra: Additional info as a comma separated string (see extra_args above)
 * children: Dictionary defining a nested dictionary where the key is the name
   of the sub-directory and the value is the item info (see items above)
-* nicknames: List of METplus variable names (with app name excluded) to also
+* nicknames: List of METplus variable names to also
   search and use if it is set. For example, the GridStat variable mask.poly is
   set by the METplus config variable GRID_STAT_MASK_POLY. However, in older
   versions of the METplus wrappers, the variable used was
   GRID_STAT_VERIFICATION_MASK_TEMPLATE. To preserve support for this name, the
-  nickname can be set to ['VERIFICATION_MASK_TEMPLATE'] and the old variable
+  nickname can be set to [f'{self.app_name.upper()}_VERIFICATION_MASK_TEMPLATE'] and the old variable
   will be checked if GRID_STAT_MASK_POLY is not set.
 
 Values must be set to None to preserve the order.
@@ -319,8 +319,8 @@ a function is typically used to handle it. For example, this function is in
 CompareGriddedWrapper and is used by GridStat, PointStat, and EnsembleStat::
 
     def handle_climo_cdf_dict(self):
-        self.handle_met_config_dict('climo_cdf', {
-            'cdf_bins': ('float', None, None, ['CLIMO_CDF_BINS']),
+        self.add_met_config_dict('climo_cdf', {
+            'cdf_bins': ('float', None, None, [f'{self.app_name.upper()}_CLIMO_CDF_BINS']),
             'center_bins': 'bool',
             'write_bins': 'bool',
         })
@@ -329,30 +329,30 @@ This function handles setting the climo_cdf dictionary. The METplus config
 variable that fits the format {APP_NAME}_{DICTIONARY_NAME}_{VARIABLE_NAME},
 i.e. GRID_STAT_CLIMO_CDF_CDF_BINS for GridStat's climo_cdf.cdf_bins, is
 quieried first. However, this default name is a little redundant, so adding
-the nickname 'CLIMO_CDF_BINS' allows the user to set the variable
+the nickname 'GRID_STAT_CLIMO_CDF_BINS' allows the user to set the variable
 GRID_STAT_CLIMO_CDF_BINS instead.
 
 There are many MET config dictionaries that only contain beg and end to define
-a window. A function in CommandBuilder called handle_met_config_window can be
+a window. A function in CommandBuilder called add_met_config_window can be
 used to easily set these variable by only supplying the name of the MET
 dictionary variable.
 
 ::
 
-    def handle_met_config_window(self, dict_name):
+    def add_met_config_window(self, dict_name):
         """! Handle a MET config window dictionary. It is assumed that
         the dictionary only contains 'beg' and 'end' entries that are integers.
 
         @param dict_name name of MET dictionary
         """
-        self.handle_met_config_dict(dict_name, {
+        self.add_met_config_dict(dict_name, {
             'beg': 'int',
             'end': 'int',
         })
 
 This can be called from any wrapper, i.e. TCGen::
 
-    self.handle_met_config_window('fcst_hr_window')
+    self.add_met_config_window('fcst_hr_window')
 
 This will check if TC_GEN_FCST_HR_WINDOW_BEGIN (or TC_GEN_FCST_HR_WINDOW_BEG)
 and TC_GEN_FCST_HR_WINDOW_END are set and override fcst_hr_window.beg and/or
@@ -383,5 +383,5 @@ handle_climo_dict, handle_mask, and handle_interp_dict.
         if uses_field:
             items['field'] = ('string', 'remove_quotes')
 
-        self.handle_met_config_dict('interp', items)
+        self.add_met_config_dict('interp', items)
 

docs/Release_Guide/release_steps/create_release_on_github.rst

@@ -13,7 +13,7 @@ Create Release on GitHub
   https://|projectRepo|.readthedocs.io/en/vX.Y.Z-betaN/Users_Guide/release-notes.html
   (Note: the URL will not be active until the release is created)
 
-* Attach a PDF of the |projectRepo| User's Guide, if available.
+* Add a link to the PDF of the |projectRepo| User's Guide, if available.
   The PDF can be downloaded from ReadTheDocs if it is available, i.e.
   https://|projectRepo|.readthedocs.io/_/downloads/en/vX.Y.Z-betaN/pdf/
   (Note: the URL will not be active until the release is created)